<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/15/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/15/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5094.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5094.html" class="post-title-link" itemprop="url">爬虫代理哪家强？十大付费代理详细对比评测出炉！</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>随着大数据时代的到来，爬虫已经成了获取数据的必不可少的方式，做过爬虫的想必都深有体会，爬取的时候莫名其妙 IP 就被网站封掉了，毕竟各大网站也不想自己的数据被轻易地爬走。 对于爬虫来说，为了解决封禁 IP 的问题，一个有效的方式就是使用代理，使用代理之后可以让爬虫伪装自己的真实 IP，如果使用大量的随机的代理进行爬取，那么网站就不知道是我们的爬虫一直在爬取了，这样就有效地解决了反爬的问题。 那么问题来了，使用什么代理好呢？这里指的代理一般是 HTTP 代理，主要用于数据爬取。现在打开搜索引擎一搜 HTTP 代理，免费的、付费的太多太多品牌，我们该如何选择呢？看完这一篇文章，想必你心中就有了答案。 对于免费代理，其实想都不用想了，可用率能超过 10% 就已经是谢天谢地了。真正靠谱的代理还是需要花钱买的，那这么多家到底哪家可用率高？哪家响应速度快？哪家比较稳定？哪家性价比比较高？为此，我对市面上比较流行的多家付费代理针对可用率、爬取速度、爬取稳定性、价格、安全性、请求限制等做了详细的评测，让我们来一起看一下到底哪家更强！</p>
                  <h2 id="测评范围"><a href="#测评范围" class="headerlink" title="测评范围"></a>测评范围</h2>
                  <h3 id="免费代理"><a href="#免费代理" class="headerlink" title="免费代理"></a>免费代理</h3>
                  <p>在这里我主要测试的是付费代理，免费代理可用率太低，几乎不会超过 10%，但为了作为对比，我选取了西刺免费代理进行了测试。</p>
                  <h3 id="付费代理"><a href="#付费代理" class="headerlink" title="付费代理"></a>付费代理</h3>
                  <p>付费代理我选取了站大爷、芝麻 HTTP 代理、太阳 HTTP 代理、讯代理、快代理、蘑菇代理、阿布云代理、全网代理、云代理、大象代理、多贝云进行了对比评测，购买了他们的各个不同级别的套餐使用同样的网络环境进行了测评，详情如下：</p>
                  <p> 代理商家</p>
                  <p> 套餐类型</p>
                  <p>官方网站</p>
                  <p>芝麻 HTTP 代理</p>
                  <p> 默认版</p>
                  <p><a href="http://www.zhimaruanjian.com/" target="_blank" rel="noopener">http://www.zhimaruanjian.com/</a></p>
                  <p>阿布云代理</p>
                  <p> 专业版</p>
                  <p><a href="https://www.abuyun.com" target="_blank" rel="noopener">https://www.abuyun.com</a></p>
                  <p> 动态版</p>
                  <p> 经典版</p>
                  <p> 大象代理</p>
                  <p> 个人版</p>
                  <p><a href="http://www.daxiangdaili.com" target="_blank" rel="noopener">http://www.daxiangdaili.com</a></p>
                  <p> 专业版</p>
                  <p> 企业版</p>
                  <p>全网代理</p>
                  <p> 普通版</p>
                  <p><a href="http://www.goubanjia.com" target="_blank" rel="noopener">http://www.goubanjia.com</a></p>
                  <p> 动态版</p>
                  <p> 快代理</p>
                  <p> VIP 套餐</p>
                  <p><a href="https://www.kuaidaili.com" target="_blank" rel="noopener">https://www.kuaidaili.com</a></p>
                  <p> 蘑菇代理</p>
                  <p> 默认版</p>
                  <p><a href="http://www.mogumiao.com" target="_blank" rel="noopener">http://www.mogumiao.com</a></p>
                  <p> 太阳 HTTP 代理</p>
                  <p> 默认版</p>
                  <p><a href="http://http.taiyangruanjian.com" target="_blank" rel="noopener">http://http.taiyangruanjian.com</a></p>
                  <p> 讯代理</p>
                  <p> 优质代理</p>
                  <p><a href="http://www.xdaili.cn" target="_blank" rel="noopener">http://www.xdaili.cn</a></p>
                  <p> 混播代理</p>
                  <p> 独享代理</p>
                  <p> 云代理</p>
                  <p> VIP 套餐</p>
                  <p><a href="http://www.ip3366.net" target="_blank" rel="noopener">http://www.ip3366.net</a></p>
                  <p> 站大爷代理</p>
                  <p> 普通代理</p>
                  <p><a href="http://ip.zdaye.com" target="_blank" rel="noopener">http://ip.zdaye.com</a></p>
                  <p> 短效优质代理</p>
                  <p>多贝云代理</p>
                  <p> 套餐一</p>
                  <p><a href="http://dobel.cn/" target="_blank" rel="noopener">http://dobel.cn/</a></p>
                  <p> 套餐二</p>
                  <p> 套餐三</p>
                  <p>注：其中蘑菇代理、太阳 HTTP 代理、芝麻 HTTP 代理的默认版表示此网站只有这一种代理，不同套餐仅是时长区别，代理质量没有差别。 嗯，我把上面的套餐全部买了一遍，以供下面的评测使用。</p>
                  <h2 id="测评目标"><a href="#测评目标" class="headerlink" title="测评目标"></a>测评目标</h2>
                  <p>本次测评主要分析代理的可用率、响应速度、稳定性、价格、安全性、使用频率等因素，下面我们来一一进行说明。</p>
                  <h3 id="可用率"><a href="#可用率" class="headerlink" title="可用率"></a>可用率</h3>
                  <p>可用率就是提取的这些代理中可以正常使用的比率。假如我们无法使用这个代理请求某个网站或者访问超时，那么就代表这个代理不可用，在这里我的测试样本大小为 500，即提取 500 个代理，看看里面可用的比率多少。</p>
                  <h3 id="响应速度"><a href="#响应速度" class="headerlink" title="响应速度"></a>响应速度</h3>
                  <p>响应速度可以用耗费时间来衡量，即计算使用这个代理请求网站一直到得到响应所耗费的时间。时间越短，证明代理的响应速度越快，这里同样是 500 个样本，计算时只对正常可用的代理做统计，计算耗费时间的平均值。</p>
                  <h3 id="稳定性"><a href="#稳定性" class="headerlink" title="稳定性"></a>稳定性</h3>
                  <p>由于爬虫时我们需要使用大量代理，如果一个代理响应速度特别快，很快就能得到响应，而下一次请求使用的代理响应速度特别慢，等了三十秒才得到响应，那势必会影响爬取效率，所以我们需要看下商家提供的这些代理稳定性怎样，总不能这一个特别快，下一个又慢的不行。所以这里我们需要统计一下耗费时间的方差，方差越大，证明稳定性越差。</p>
                  <h3 id="价格"><a href="#价格" class="headerlink" title="价格"></a>价格</h3>
                  <p>价格，这个当然是需要考虑的内容，如果一个代理不论是响应速度还是稳定性都特别不错，但是价格非常非常高，这也是不可接受的。</p>
                  <h3 id="安全性"><a href="#安全性" class="headerlink" title="安全性"></a>安全性</h3>
                  <p>这的确也是需要考虑的因素，比如一旦不小心把代理提取的 API 泄露出去了，别人就肆意使用我们的 API 提取代理使用，而一直耗费的是我们的套餐。另外一旦别人通过某些手段获取了我们的代理列表，而这些代理是没有安全验证的，这也会导致别人偷偷使用我们的代理。在生产环境上，这方面尤其需要注意。</p>
                  <h3 id="使用频率"><a href="#使用频率" class="headerlink" title="使用频率"></a>使用频率</h3>
                  <p>有些代理套餐在 API 调用提取代理时有频率限制，有的代理套餐则会限制请求频率，这些因素都会或多或少影响爬虫的效率，这部分因素我们也需要考虑进来。</p>
                  <h2 id="测评标准"><a href="#测评标准" class="headerlink" title="测评标准"></a>测评标准</h2>
                  <p>要做标准的测评，那就必须在标准的测评环境下进行，且尽可能排除一些杂项的干扰，如网络波动、传输延迟等一系列的影响。</p>
                  <h3 id="主机选取"><a href="#主机选取" class="headerlink" title="主机选取"></a>主机选取</h3>
                  <p>由于我的个人笔记本是使用 WiFi 上网的，所以可能会有网络波动，而且实际带宽其实并不太好把控，因此它并不适合来做标准评测使用。评测需要在一个网络稳定的条件下进行，而且多个代理的评测环境必须相同，在此我选择了一台腾讯云主机作为测试，主机配置如下：</p>
                  <p> 参数名</p>
                  <p> 参数值</p>
                  <p> 操作系统</p>
                  <p>Ubuntu 16.04.1 LTS (GNU/Linux 4.4.0-53-generic x86_64)</p>
                  <p> 带宽</p>
                  <p>5 Mbps</p>
                  <p> 核心数</p>
                  <p> 2</p>
                  <p> 内存</p>
                  <p> 4GB</p>
                  <p>Python 版本</p>
                  <p> 3.5.2</p>
                  <p>这样我们就可以保证一个标准统一的测试环境了。</p>
                  <h3 id="现取现测"><a href="#现取现测" class="headerlink" title="现取现测"></a>现取现测</h3>
                  <p>另外在评测时还需要遵循一个原则，那就是现取现测，即取一个测一个。现在很多付费代理网站都提供了 API 接口，我们可以一次性提取多个代理，但是这样会导致一个问题，每个代理在提取出来的时候，商家是会尽量保证它的可用性的，但过一段时间，这个代理可能就不好用了，所以假如我们一次性提取出来了 100 个代理，但是这 100 个代理并没有同时参与测试，后面的代理就会经历一个的等待期，过一段时间再测这些代理的话，肯定会影响后半部分代理的有效性，所以这里我们将提取的数量统一设置成 1，即请求一次接口获取一个代理，然后立即进行测试，这样可以保证测试的公平性，排除了不同代理有效期的干扰。</p>
                  <h3 id="时间计算"><a href="#时间计算" class="headerlink" title="时间计算"></a>时间计算</h3>
                  <p>由于我们有一项是测试代理的响应速度，所以我们需要计算程序请求之前和得到响应之后的时间差，这里我们使用的测试 Python 库是 requests，所以我们就计算发起请求和得到响应之间的时间差即可，时间计算方法如下所示：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">start_time = <span class="built_in">time</span>.<span class="built_in">time</span>()</span><br><span class="line">requests.<span class="keyword">get</span>(test_url, <span class="keyword">timeout</span>=<span class="keyword">timeout</span>, proxies=proxies)</span><br><span class="line">end_time = <span class="built_in">time</span>.<span class="built_in">time</span>()</span><br><span class="line">used_time = end_time - start_time</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 used_time 就是使用代理请求的耗时，这样测试的就仅仅是发起请求到得到响应的时间。</p>
                  <h3 id="测试链接"><a href="#测试链接" class="headerlink" title="测试链接"></a>测试链接</h3>
                  <p>测试时我们也需要使用一个稳定的且没有反爬虫的链接，这样可以排除服务器的干扰，这里我们使用百度来作为测试目标。</p>
                  <h3 id="超时限制"><a href="#超时限制" class="headerlink" title="超时限制"></a>超时限制</h3>
                  <p>在测试时免不了的会遇到代理请求超时的问题，所以这里我们也需要统一一个超时时间，这里设置为 60 秒，如果使用代理请求百度，60 秒还没有得到响应，那就视为该代理无效。</p>
                  <h3 id="测试数量"><a href="#测试数量" class="headerlink" title="测试数量"></a>测试数量</h3>
                  <p>要做测评，那么样本不能太小，如只有十几次测试是不能轻易下结论的，这里我选取了一个适中的测评数量 500，即每个套餐获取 500 个代理进行测试。</p>
                  <h2 id="测评过程"><a href="#测评过程" class="headerlink" title="测评过程"></a>测评过程</h2>
                  <p>嗯，测评过程这边主要说一下测评的代码逻辑，首先测的时候是取一个测一个的，所以这里定义了一个 test_proxy() 方法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">test_url = <span class="string">'https://www.baidu.com/'</span></span><br><span class="line">timeout = 60</span><br><span class="line"></span><br><span class="line">def test_proxy(proxy):</span><br><span class="line">    try:</span><br><span class="line">        proxies = &#123;</span><br><span class="line">            <span class="string">'https'</span>: <span class="string">'http://'</span> + proxy</span><br><span class="line">        &#125;</span><br><span class="line">        start_time = time.time()</span><br><span class="line">        requests.<span class="builtin-name">get</span>(test_url, <span class="attribute">timeout</span>=timeout, <span class="attribute">proxies</span>=proxies)</span><br><span class="line">        end_time = time.time()</span><br><span class="line">        used_time = end_time - start_time</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'Proxy Valid'</span>, <span class="string">'Used Time:'</span>, used_time)</span><br><span class="line">        return <span class="literal">True</span>, used_time</span><br><span class="line">    except (ProxyError, ConnectTimeout, SSLError, ReadTimeout, ConnectionError):</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'Proxy Invalid:'</span>, proxy)</span><br><span class="line">        return <span class="literal">False</span>, None</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要传入一个参数 proxy，代表一个代理，即 IP 加端口组成的代理，然后这里使用了 requests 的 proxies 参数传递给 get() 方法。对于代理无效的检测，这里判断了 ProxyError, ConnectTimeout, SSLError, ReadTimeout, ConnectionError 这几种异常，如果发生了这些异常统统视为代理无效，返回错误。如果在 timeout 60 秒内得到了响应，那么就计算其耗费时间并返回。 在主程序里，就是获取 API 然后统计结果了，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">max = 500</span><br><span class="line"></span><br><span class="line">def main():</span><br><span class="line">    <span class="builtin-name">print</span>(<span class="string">'Testing'</span>)</span><br><span class="line">    used_time_list = []</span><br><span class="line">    valid_count = 0</span><br><span class="line">    total_count = 0</span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        flag, result = get_page(api_url)</span><br><span class="line">        <span class="keyword">if</span> flag:</span><br><span class="line">           <span class="built_in"> proxy </span>= result.strip()</span><br><span class="line">            <span class="keyword">if</span> is_proxy(proxy):</span><br><span class="line">                total_count += 1</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'Testing proxy'</span>, proxy)</span><br><span class="line">                test_flag, test_result = test_proxy(<span class="attribute">proxy</span>=proxy)</span><br><span class="line">                <span class="keyword">if</span> test_flag:</span><br><span class="line">                    valid_count += 1</span><br><span class="line">                    used_time_list.append(test_result)</span><br><span class="line">                stats_result(used_time_list, valid_count, total_count)</span><br><span class="line">        time.sleep(wait)</span><br><span class="line">        <span class="keyword">if</span> total_count == max:</span><br><span class="line">            break</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里加了一些判断，如 is_proxy() 方法判断了获取的是不是符合有效的代理规则，即判断它是不是 IP 加端口的形式，这样可以排除 API 返回一些错误信息的干扰。另外这里设置了 total_count 和 valid_count 变量，只有符合代理规则的代理参与了测试，这样才算一次有效测试，total_count 加一，如果测试可用，那么 valid_count 加一并记录耗费时间。最后调用了 stats_results 方法进行了统计：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">stats_result</span><span class="params">(used_time_list, valid_count, total_count)</span>:</span></span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> used_time_list <span class="keyword">or</span> <span class="keyword">not</span> total_count:</span><br><span class="line">        <span class="keyword">return</span></span><br><span class="line">    used_time_array = np.asarray(used_time_list, np.float32)</span><br><span class="line">    print(<span class="string">'Total Count:'</span>, total_count,</span><br><span class="line">          <span class="string">'Valid Count:'</span>, valid_count,</span><br><span class="line">          <span class="string">'Valid Percent: %.2f%%'</span> % (valid_count * <span class="number">100.0</span> / total_count),</span><br><span class="line">          <span class="string">'Used Time Mean:'</span>, used_time_array.mean(),</span><br><span class="line">          <span class="string">'Used Time Var'</span>, used_time_array.var())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用了 Numpy 来统计了耗费时间的均值和方差，分别反映代理的响应速度和稳定性。 嗯，就这样，利用这个方法我对各个不同的代理套餐逐一进行了测试。</p>
                  <h2 id="测评结果"><a href="#测评结果" class="headerlink" title="测评结果"></a>测评结果</h2>
                  <p>经过测评，初步得到如下统计结果：</p>
                  <p> 代理商家</p>
                  <p> 套餐类型</p>
                  <p> 测试次数</p>
                  <p> 有效次数</p>
                  <p> 可用率</p>
                  <p> 响应时间均值</p>
                  <p> 响应时间方差</p>
                  <p> 芝麻 HTTP 代理</p>
                  <p> 默认版</p>
                  <p> 500</p>
                  <p> 495</p>
                  <p> 99.00%</p>
                  <p> 0.916853</p>
                  <p> 1.331989</p>
                  <p>阿布云代理</p>
                  <p> 专业版</p>
                  <p> 500</p>
                  <p> 452</p>
                  <p> 90.40%</p>
                  <p>0.68770707</p>
                  <p>1.1477163</p>
                  <p> 动态版</p>
                  <p> 500</p>
                  <p> 494</p>
                  <p> 98.80%</p>
                  <p>1.83994</p>
                  <p>6.0491614</p>
                  <p> 经典版</p>
                  <p> 500</p>
                  <p> 499</p>
                  <p> 99.80%</p>
                  <p>0.49301904</p>
                  <p>0.25724468</p>
                  <p> 大象代理</p>
                  <p> 个人版</p>
                  <p> 500</p>
                  <p> 238</p>
                  <p> 47.60%</p>
                  <p>5.340489</p>
                  <p>78.56444</p>
                  <p> 专业版</p>
                  <p> 500</p>
                  <p> 284</p>
                  <p> 56.80%</p>
                  <p>6.87078</p>
                  <p>105.7984</p>
                  <p> 企业版</p>
                  <p> 500</p>
                  <p> 259</p>
                  <p> 51.80%</p>
                  <p>6.3081837</p>
                  <p>121.08402</p>
                  <p>全网代理</p>
                  <p> 普通版</p>
                  <p> 500</p>
                  <p> 220</p>
                  <p> 44.00%</p>
                  <p>5.584057</p>
                  <p>47.442596</p>
                  <p> 动态版</p>
                  <p> 500</p>
                  <p> 485</p>
                  <p> 97.00%</p>
                  <p>2.776973</p>
                  <p>17.568045</p>
                  <p>快代理</p>
                  <p> VIP 套餐</p>
                  <p> 500</p>
                  <p> 178</p>
                  <p> 35.60%</p>
                  <p>16.636587</p>
                  <p>221.69661</p>
                  <p>蘑菇代理</p>
                  <p> 默认版</p>
                  <p> 500</p>
                  <p> 497</p>
                  <p> 99.40%</p>
                  <p>1.0985725</p>
                  <p>9.532586</p>
                  <p>太阳 HTTP 代理</p>
                  <p> 默认版</p>
                  <p> 500</p>
                  <p> 400</p>
                  <p> 80.00%</p>
                  <p>1.2522483</p>
                  <p>12.662229</p>
                  <p>讯代理</p>
                  <p> 优质代理</p>
                  <p> 500</p>
                  <p> 495</p>
                  <p> 99.00%</p>
                  <p>1.0512681</p>
                  <p>6.4247565</p>
                  <p> 混播代理</p>
                  <p> 500</p>
                  <p> 494</p>
                  <p> 98.80%</p>
                  <p>1.0664985</p>
                  <p>6.451699</p>
                  <p> 独享代理</p>
                  <p> 500</p>
                  <p> 500</p>
                  <p> 100%</p>
                  <p> 0.7056521</p>
                  <p> 0.35416448</p>
                  <p>多贝云代理</p>
                  <p> 套餐一</p>
                  <p> 500</p>
                  <p> 500</p>
                  <p> 100.00%</p>
                  <p>0.6580079</p>
                  <p>0.199466</p>
                  <p> 套餐二</p>
                  <p> 500</p>
                  <p> 500</p>
                  <p> 100.00%</p>
                  <p>0.510749</p>
                  <p>0.022519</p>
                  <p> 套餐三</p>
                  <p> 500</p>
                  <p> 499</p>
                  <p> 99.80%</p>
                  <p> 0.6785444</p>
                  <p> 0.5197055</p>
                  <p> 云代理</p>
                  <p> VIP 套餐</p>
                  <p> 500</p>
                  <p> 489</p>
                  <p> 97.80%</p>
                  <p> 3.4216988</p>
                  <p> 38.120296</p>
                  <p> 站大爷代理</p>
                  <p> 普通代理</p>
                  <p> 500</p>
                  <p> 92</p>
                  <p> 18.40%</p>
                  <p> 5.067193</p>
                  <p>66.12128</p>
                  <p> 短效优质代理</p>
                  <p> 500</p>
                  <p> 488</p>
                  <p> 97.60%</p>
                  <p> 1.5625348</p>
                  <p>8.121197</p>
                  <p> 西刺代理</p>
                  <p> 免费</p>
                  <p> 500</p>
                  <p> 31</p>
                  <p> 6.2%</p>
                  <p>9.712833</p>
                  <p>95.09569</p>
                  <p>注：</p>
                  <ul>
                    <li><strong>表中的响应时间方差越大，代表稳定性越低。</strong></li>
                    <li><strong>阿布云代理经典版方差较小是因为它是长时间锁定了同一个 IP，因此极其稳定，但每秒最大请求默认 5 次。</strong></li>
                    <li><strong>多贝云代理套餐一二方差较小是因为它是长时间锁定了同一个 IP，因此极其稳定，但每秒最大请求默认 20 次。</strong></li>
                  </ul>
                  <h2 id="测评分析"><a href="#测评分析" class="headerlink" title="测评分析"></a>测评分析</h2>
                  <p>下面我们将从各个方面分析一下各个套餐的优劣。</p>
                  <h3 id="可用率-1"><a href="#可用率-1" class="headerlink" title="可用率"></a>可用率</h3>
                  <p>通过可用率统计，我们可以发现可用率较高的代理套餐有：</p>
                  <p> 级别</p>
                  <p> 套餐</p>
                  <p>描述</p>
                  <p> 第一梯队</p>
                  <p> 多贝云代理、讯代理独享代理、阿布云代理经典版、蘑菇代理、芝麻 HTTP 代理、讯代理优质代理</p>
                  <p> 可用率 99% 以上</p>
                  <p> 第二梯队</p>
                  <p> 阿布云代理动态版、讯代理混播代理、云代理、站大爷短效优质代理、全网代理动态版、阿布云代理专业版</p>
                  <p> 可用率 99% 以下，90% 以上</p>
                  <p> 第三梯队</p>
                  <p> 太阳 HTTP 代理、大象代理专业版、大象代理企业版</p>
                  <p> 可用率 90% 以下，50% 以上</p>
                  <p> 第四梯队</p>
                  <p> 大象代理个人版、全网代理普通版、快代理</p>
                  <p>可用率 50% 以下，20% 以上</p>
                  <p> 第五梯队</p>
                  <p>站大爷普通代理、西刺代理</p>
                  <p> 可用率 20% 以下</p>
                  <h3 id="响应速度-1"><a href="#响应速度-1" class="headerlink" title="响应速度"></a>响应速度</h3>
                  <p>通过平均响应速度判别，我们可以发现响应速度较快的代理套餐有：</p>
                  <p> 级别</p>
                  <p> 套餐</p>
                  <p> 描述</p>
                  <p> 第一梯队</p>
                  <p> 多贝云代理、阿布云代理经典版、阿布云代理专业版、讯代理独享代理、芝麻 HTTP 代理</p>
                  <p>响应时间 1s 以内</p>
                  <p> 第二梯队</p>
                  <p> 讯代理优质代理、讯代理混播代理、蘑菇代理、太阳代理、站大爷短效优质代理、阿布云代理动态版</p>
                  <p>响应时间 1s 以上，2s 以内</p>
                  <p> 第三梯队</p>
                  <p>全网代理动态版、云代理</p>
                  <p>响应时间 2s 以上，5s 以内</p>
                  <p> 第四梯队</p>
                  <p>站大爷普通代理、大象代理个人版、全网代理普通版、大象代理企业版、大象代理专业版、西刺代理</p>
                  <p>响应时间 5s 以上，10s 以内</p>
                  <p> 第五梯队</p>
                  <p>快代理</p>
                  <p>响应时间 10s 以上</p>
                  <h3 id="稳定性-1"><a href="#稳定性-1" class="headerlink" title="稳定性"></a>稳定性</h3>
                  <p>通过平均响应速度方差分析，我们可以发现稳定性较高的代理套餐有：</p>
                  <p> 级别</p>
                  <p> 套餐</p>
                  <p> 描述</p>
                  <p> 第一梯队</p>
                  <p>多贝云代理、阿布云代理经典版、讯代理独享代理、阿布云代理专业版、芝麻 HTTP 代理</p>
                  <p> 方差 3 以内</p>
                  <p> 第二梯队</p>
                  <p>阿布云代理动态版、讯代理优质代理、讯代理混播代理、站大爷短效优质代理、蘑菇代理</p>
                  <p>方差 10 以内，3 以上</p>
                  <p> 第三梯队</p>
                  <p>太阳HTTP代理、全网代理动态版、云代理、全网代理普通版、站大爷普通代理、大象代理个人版、西刺代理</p>
                  <p> 方差 100 以内，10 以上</p>
                  <p> 第四梯队</p>
                  <p> 大象代理专业版、大象代理企业版、快代理</p>
                  <p> 方差 100 以上</p>
                  <h3 id="价格-1"><a href="#价格-1" class="headerlink" title="价格"></a>价格</h3>
                  <p>我们可以先看一下各个套餐的价格：</p>
                  <p> 代理商家</p>
                  <p> 套餐类型</p>
                  <p>价格描述</p>
                  <p> 价格 URL</p>
                  <p> 备注</p>
                  <p>芝麻 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>￥40/周 ￥114/月</p>
                  <p><a href="http://www.zhimaruanjian.com/pay/" target="_blank" rel="noopener">http://www.zhimaruanjian.com/pay/</a></p>
                  <p>另有包量套餐、定期有优惠活动，可领免费 IP，可免费试用</p>
                  <p>阿布云代理</p>
                  <p> 专业版</p>
                  <p>￥1/时 ￥16/天 ￥108/周 ￥429/月</p>
                  <p><a href="https://www.abuyun.com/" target="_blank" rel="noopener">https://www.abuyun.com/</a></p>
                  <p> 每秒请求只有5个，多加每秒请求1个需要 1￥0.5/月，￥90 /年</p>
                  <p> 动态版</p>
                  <p>￥1/时 ￥16/天 ￥108/周 ￥429/月</p>
                  <p> 经典版</p>
                  <p>￥1/时 ￥16/天 ￥108/周 ￥429/月</p>
                  <p>多贝云代理</p>
                  <p> 套餐一</p>
                  <p>￥500/月</p>
                  <p><a href="http://dobel.cn/" target="_blank" rel="noopener">http://dobel.cn/</a></p>
                  <p> 套餐一二默认每秒最多请求 20 次 套餐三每秒默认最多请求 5次，最多都可购买到 100 个请求数</p>
                  <p> 套餐二</p>
                  <p>￥600/月</p>
                  <p> 套餐三</p>
                  <p>￥425/月</p>
                  <p> 大象代理</p>
                  <p> 个人版</p>
                  <p>￥9/天 ￥98/月</p>
                  <p><a href="http://www.daxiangdaili.com/" target="_blank" rel="noopener">http://www.daxiangdaili.com/</a></p>
                  <p>好评可送时长</p>
                  <p> 专业版</p>
                  <p>￥19/天 ￥198/月</p>
                  <p> 企业版</p>
                  <p>￥49/天 ￥498/月</p>
                  <p>全网代理</p>
                  <p> 普通版</p>
                  <p>￥9/天 ￥35/周 ￥93/月 ￥500/年</p>
                  <p><a href="http://www.goubanjia.com/buy/high.shtml" target="_blank" rel="noopener">http://www.goubanjia.com/buy/high.shtml</a></p>
                  <p> 动态版</p>
                  <p>￥10/天 ￥160/月 ￥1250/年</p>
                  <p><a href="http://www.goubanjia.com/buy/dynamic.shtml" target="_blank" rel="noopener">http://www.goubanjia.com/buy/dynamic.shtml</a></p>
                  <p> 快代理</p>
                  <p> VIP 套餐</p>
                  <p>￥20/天 ￥60/周 ￥200/月 ￥2000/年</p>
                  <p><a href="https://www.kuaidaili.com/pricing" target="_blank" rel="noopener">https://www.kuaidaili.com/pricing</a></p>
                  <p>有普通、VIP、SVIP、专业版可选</p>
                  <p> 蘑菇代理</p>
                  <p> 默认版</p>
                  <p>￥6/天 ￥169/月 ￥1699/年</p>
                  <p><a href="http://www.mogumiao.com/buy" target="_blank" rel="noopener">http://www.mogumiao.com/buy</a></p>
                  <p>另有包量套餐可选购，可免费试用</p>
                  <p> 太阳 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>￥60/周 ￥198/月 ￥498/季 ￥1590/年</p>
                  <p><a href="http://http.taiyangruanjian.com/newrecharge/" target="_blank" rel="noopener">http://http.taiyangruanjian.com/newrecharge/</a></p>
                  <p>另有保量套餐可选购，可领免费 IP，可免费试用</p>
                  <p> 讯代理</p>
                  <p> 优质代理</p>
                  <p>￥9/天 ￥210/月 ￥2100/年</p>
                  <p><a href="http://www.xdaili.cn/buyproxy" target="_blank" rel="noopener">http://www.xdaili.cn/buyproxy</a></p>
                  <p> 可免费试用</p>
                  <p> 混播代理</p>
                  <p>￥29/天 ￥729/月 ￥6999/年</p>
                  <p> 独享代理</p>
                  <p>￥9/天 ￥210/月 ￥2100/年</p>
                  <p> 云代理</p>
                  <p> VIP 套餐</p>
                  <p>￥10/天 ￥120/月 ￥599/年</p>
                  <p><a href="http://www.ip3366.net/pricing/" target="_blank" rel="noopener">http://www.ip3366.net/pricing/</a></p>
                  <p> 另有普通套餐可选</p>
                  <p> 站大爷代理</p>
                  <p> 普通代理</p>
                  <p>￥8/天 ￥80/月 ￥720/年</p>
                  <p><a href="http://ip.zdaye.com/buy.html" target="_blank" rel="noopener">http://ip.zdaye.com/buy.html</a></p>
                  <p> 另有私密代理可选</p>
                  <p> 短效优质代理</p>
                  <p>￥17/天 ￥475/月 ￥4569/年</p>
                  <p><a href="http://ip.zdaye.com/ShortProxy.html" target="_blank" rel="noopener">http://ip.zdaye.com/ShortProxy.html</a></p>
                  <p>按照包月的价格，我们可以统一对比如下：</p>
                  <p> 级别</p>
                  <p> 套餐</p>
                  <p> 描述</p>
                  <p> 第一梯队</p>
                  <p> 多贝云代理、大象代理企业版、站大爷短效优质代理、阿布云代理、芝麻 HTTP代理</p>
                  <p> 包月大于 400</p>
                  <p> 第二梯队</p>
                  <p> 讯代理混播代理</p>
                  <p> 包月小于 400，大于 300</p>
                  <p> 第三梯队</p>
                  <p> 讯代理优质代理、讯代理独享代理、快代理</p>
                  <p> 包月小于 300，大于 200</p>
                  <p> 第四梯队</p>
                  <p> 太阳 HTTP 代理、大象代理专业版、蘑菇代理、全网代理动态版、云代理</p>
                  <p> 包月小于 200，大于 100</p>
                  <p> 第五梯队</p>
                  <p> 大象代理个人版、全网代理、站大爷普通代理</p>
                  <p> 包月小于 100</p>
                  <h3 id="安全性-1"><a href="#安全性-1" class="headerlink" title="安全性"></a>安全性</h3>
                  <p>对于安全性，此处主要考虑提取 API 是否有访问验证，使用代理时是否有访问验证，即可以通过设置白名单来控制哪些可以使用。 其中只有芝麻 HTTP 代理、太阳 HTTP 代理默认使用了白名单限制，即只有将使用 IP 添加到白名单才可以使用，可以有效控制使用权限。 另外阿布云代理提供了隧道代理验证，只有成功配置了用户名和密码才可以正常使用。 所以在此归纳如下：</p>
                  <p> 级别</p>
                  <p> 套餐</p>
                  <p> 描述</p>
                  <p> 第一梯队</p>
                  <p>芝麻 HTTP 代理、太阳 HTTP 代理、阿布云代理、多贝云代理</p>
                  <p>默认使用了白名单控制或隧道代理验证</p>
                  <p> 第二梯队</p>
                  <p> 其他</p>
                  <p>可直接使用</p>
                  <h3 id="调取频率"><a href="#调取频率" class="headerlink" title="调取频率"></a>调取频率</h3>
                  <p>不同的接口具有不同的 API 调用频率限制，归纳如下：</p>
                  <p> 代理商家</p>
                  <p> 套餐类型</p>
                  <p>调取频率限制</p>
                  <p>芝麻 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>1秒</p>
                  <p>阿布云代理</p>
                  <p> 专业版</p>
                  <p>无需获取</p>
                  <p> 动态版</p>
                  <p>无需获取</p>
                  <p> 经典版</p>
                  <p>无需获取</p>
                  <p>多贝云代理</p>
                  <p> 套餐一</p>
                  <p>无需获取</p>
                  <p> 套餐二</p>
                  <p>无需获取</p>
                  <p> 套餐三</p>
                  <p>无需获取</p>
                  <p> 大象代理</p>
                  <p> 个人版</p>
                  <p>1秒</p>
                  <p> 专业版</p>
                  <p>1秒</p>
                  <p> 企业版</p>
                  <p>无限制</p>
                  <p>全网代理</p>
                  <p> 普通版</p>
                  <p>无限制</p>
                  <p> 动态版</p>
                  <p>100毫秒</p>
                  <p>快代理</p>
                  <p> VIP 套餐</p>
                  <p>200毫秒</p>
                  <p>蘑菇代理</p>
                  <p> 默认版</p>
                  <p>5秒</p>
                  <p>太阳 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>1秒</p>
                  <p>讯代理</p>
                  <p> 优质代理</p>
                  <p>5秒</p>
                  <p> 混播代理</p>
                  <p>10秒</p>
                  <p> 独享代理</p>
                  <p>15秒</p>
                  <p> 云代理</p>
                  <p> VIP 套餐</p>
                  <p>无限制</p>
                  <p> 站大爷代理</p>
                  <p> 普通代理</p>
                  <p>3秒</p>
                  <p> 短效优质代理</p>
                  <p>10秒</p>
                  <p> 西刺代理</p>
                  <p> 免费</p>
                  <p>无限制</p>
                  <p>在此可以简单总结如下：</p>
                  <p>级别</p>
                  <p> 套餐</p>
                  <p> 描述</p>
                  <p> 第一梯队</p>
                  <p> 云代理、全网代理普通版、大象代理企业版、西刺代理、阿布云（调取无限制，请求默认最大 1 秒 5 请求）、多贝云（调取无限制，请求默认最大 1 秒 20 请求）</p>
                  <p> 无限制</p>
                  <p> 第二梯队</p>
                  <p> 全网代理动态版、快代理</p>
                  <p> 小于 1s</p>
                  <p> 第三梯队</p>
                  <p> 大象代理个人版、大象代理专业版、芝麻 HTTP 代理、太阳 HTTP 代理、站大爷普通代理、蘑菇代理、讯代理优质代理</p>
                  <p> 1s - 5s</p>
                  <p> 第四梯队</p>
                  <p> 讯代理混播代理、讯代理独享代理、站大爷短效优质代理</p>
                  <p> 大于 5s</p>
                  <h3 id="特色功能"><a href="#特色功能" class="headerlink" title="特色功能"></a>特色功能</h3>
                  <p>除了常规的测试之外，我这边还选取了某些套餐的与众不同之处进行说明，这些特点有的算是缺点，有的算是优点，现列举如下：</p>
                  <p> 代理</p>
                  <p> 描述</p>
                  <p> 阿布云代理 多贝云代理</p>
                  <p> 使用隧道技术实现，代理不能直接拿到，必须配置访问认证，默认 1 秒只能支持 5/20 个请求，如需更多需要付费。</p>
                  <p> 讯代理</p>
                  <p> 独享代理拨号时间略长，可用主机少，容易出现拨号失败现象，单个代理有效时长可控。</p>
                  <p> 芝麻 HTTP 代理</p>
                  <p> 必须要设置白名单才可以使用，后台可控，使用 API 提取代理不扣费，使用时才扣费。</p>
                  <h2 id="测评综合"><a href="#测评综合" class="headerlink" title="测评综合"></a>测评综合</h2>
                  <p>分项了解了各个代理套餐的可用率、响应速度、稳定性、性价比、安全性等内容之后，最后做一下总结：</p>
                  <p> 代理商家</p>
                  <p> 套餐类型</p>
                  <p>可用率</p>
                  <p> 可用率评价</p>
                  <p>响应时间均值</p>
                  <p>响应速度评价</p>
                  <p>响应时间方差</p>
                  <p>稳定性</p>
                  <p>包月价格</p>
                  <p>价格评价</p>
                  <p>安全性</p>
                  <p>访问频率限制</p>
                  <p>调取频率限制</p>
                  <p>推荐指数</p>
                  <p>芝麻 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>99%</p>
                  <p>极高</p>
                  <p>0.916853</p>
                  <p>极快</p>
                  <p>1.331989</p>
                  <p>极好</p>
                  <p>360</p>
                  <p>较高</p>
                  <p>高</p>
                  <p>无</p>
                  <p>1 秒</p>
                  <p>★★★★★</p>
                  <p>阿布云代理</p>
                  <p> 专业版</p>
                  <p>90.4%</p>
                  <p>高</p>
                  <p>0.68770707</p>
                  <p>极快</p>
                  <p>1.1477163</p>
                  <p>极好</p>
                  <p>429</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★☆</p>
                  <p> 动态版</p>
                  <p>98.8%</p>
                  <p>高</p>
                  <p>1.83994</p>
                  <p>快</p>
                  <p>6.0491614</p>
                  <p>好</p>
                  <p>429</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★★</p>
                  <p> 经典版</p>
                  <p>99.8%</p>
                  <p>极高</p>
                  <p>0.49301904</p>
                  <p>极快</p>
                  <p>0.25724468</p>
                  <p>极好</p>
                  <p>429</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★★</p>
                  <p>多贝云代理</p>
                  <p> 套餐一</p>
                  <p>100%</p>
                  <p>极高</p>
                  <p>0.658007</p>
                  <p>极快</p>
                  <p>0.199466</p>
                  <p>极好</p>
                  <p>500</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★★★</p>
                  <p> 套餐二</p>
                  <p>100%</p>
                  <p>极高</p>
                  <p>0.510748</p>
                  <p>极快</p>
                  <p>0.022519</p>
                  <p>极好</p>
                  <p>600</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★★★</p>
                  <p> 套餐三</p>
                  <p>100%</p>
                  <p>极高</p>
                  <p>0.678544</p>
                  <p>极快</p>
                  <p>0.519705</p>
                  <p>极好</p>
                  <p>425</p>
                  <p>高</p>
                  <p>高</p>
                  <p>有</p>
                  <p>无需获取</p>
                  <p>★★★★☆</p>
                  <p> 大象代理</p>
                  <p> 个人版</p>
                  <p>47.6%</p>
                  <p>低</p>
                  <p>5.340489</p>
                  <p>慢</p>
                  <p>78.56444</p>
                  <p>一般</p>
                  <p>98</p>
                  <p>低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>1 秒</p>
                  <p>★★</p>
                  <p> 专业版</p>
                  <p>56.8%</p>
                  <p>一般</p>
                  <p>6.87078</p>
                  <p>慢</p>
                  <p>105.7984</p>
                  <p>差</p>
                  <p>198</p>
                  <p>较低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>1 秒</p>
                  <p>★☆</p>
                  <p> 企业版</p>
                  <p>51.8%</p>
                  <p>一般</p>
                  <p>6.3081837</p>
                  <p>慢</p>
                  <p>121.08402</p>
                  <p>差</p>
                  <p>498</p>
                  <p>高</p>
                  <p>低</p>
                  <p>无</p>
                  <p>无限制</p>
                  <p>★</p>
                  <p>全网代理</p>
                  <p> 普通版</p>
                  <p>44%</p>
                  <p>低</p>
                  <p>5.584057</p>
                  <p>慢</p>
                  <p>47.442596</p>
                  <p>一般</p>
                  <p>93</p>
                  <p>低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>无限制</p>
                  <p>★★</p>
                  <p> 动态版</p>
                  <p>97%</p>
                  <p>高</p>
                  <p>2.776973</p>
                  <p>一般</p>
                  <p>17.568045</p>
                  <p>一般</p>
                  <p>160</p>
                  <p>较低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>100毫秒</p>
                  <p>★★★</p>
                  <p>快代理</p>
                  <p> VIP 套餐</p>
                  <p>35.6%</p>
                  <p>一般</p>
                  <p>16.636587</p>
                  <p>极慢</p>
                  <p>221.69661</p>
                  <p>差</p>
                  <p>200</p>
                  <p>中</p>
                  <p>低</p>
                  <p>无</p>
                  <p>200毫秒</p>
                  <p>☆</p>
                  <p>蘑菇代理</p>
                  <p> 默认版</p>
                  <p>99.4%</p>
                  <p>极高</p>
                  <p>1.0985725</p>
                  <p>快</p>
                  <p>9.532586</p>
                  <p>好</p>
                  <p>169</p>
                  <p>较低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>5秒</p>
                  <p>★★★★☆</p>
                  <p>太阳 HTTP 代理</p>
                  <p> 默认版</p>
                  <p>80%</p>
                  <p>一般</p>
                  <p>1.2522483</p>
                  <p>快</p>
                  <p>12.662229</p>
                  <p>一般</p>
                  <p>198</p>
                  <p>较低</p>
                  <p>高</p>
                  <p>无</p>
                  <p>1秒</p>
                  <p>★★★★</p>
                  <p>讯代理</p>
                  <p> 优质代理</p>
                  <p>99%</p>
                  <p>极高</p>
                  <p>1.0512681</p>
                  <p>快</p>
                  <p>6.4247565</p>
                  <p>好</p>
                  <p>210</p>
                  <p>中</p>
                  <p>低</p>
                  <p>无</p>
                  <p>5秒</p>
                  <p>★★★★☆</p>
                  <p> 混播代理</p>
                  <p>98.8%</p>
                  <p>高</p>
                  <p>1.0664985</p>
                  <p>快</p>
                  <p>6.451699</p>
                  <p>好</p>
                  <p>729</p>
                  <p>高</p>
                  <p>低</p>
                  <p>无</p>
                  <p>10秒</p>
                  <p>★★★☆</p>
                  <p> 独享代理</p>
                  <p>100%</p>
                  <p>极高</p>
                  <p>0.7056521</p>
                  <p>极快</p>
                  <p>0.35416448</p>
                  <p>极好</p>
                  <p>210</p>
                  <p>中</p>
                  <p>低</p>
                  <p>无</p>
                  <p>15秒</p>
                  <p>★★★★☆</p>
                  <p> 云代理</p>
                  <p> VIP 套餐</p>
                  <p>97.8%</p>
                  <p>高</p>
                  <p>3.4216988</p>
                  <p>一般</p>
                  <p>38.120296</p>
                  <p>一般</p>
                  <p>120</p>
                  <p>较低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>无限制</p>
                  <p>★★★☆</p>
                  <p> 站大爷代理</p>
                  <p> 普通代理</p>
                  <p>18.4%</p>
                  <p>极低</p>
                  <p>5.067193</p>
                  <p>慢</p>
                  <p>66.12128</p>
                  <p>一般</p>
                  <p>80</p>
                  <p>低</p>
                  <p>低</p>
                  <p>无</p>
                  <p>3秒</p>
                  <p>★☆</p>
                  <p> 短效优质代理</p>
                  <p>97.6%</p>
                  <p>高</p>
                  <p>1.5625348</p>
                  <p>快</p>
                  <p>8.121197</p>
                  <p>好</p>
                  <p>475</p>
                  <p>高</p>
                  <p>低</p>
                  <p>无</p>
                  <p>10秒</p>
                  <p>★★★☆</p>
                  <p> 西刺代理</p>
                  <p> 免费</p>
                  <p>6.2%</p>
                  <p>极低</p>
                  <p>9.712833</p>
                  <p>慢</p>
                  <p>95.09569</p>
                  <p>一般</p>
                  <p>0</p>
                  <p>免费</p>
                  <p>低</p>
                  <p>无</p>
                  <p>无限制</p>
                  <p>★</p>
                  <p>所以在综合来看比较推荐的有：芝麻代理、讯代理、阿布云、多贝云代理，详细的对比结果可以参照表格。 以上便是各家代理的详细对比测评情况，希望此文能够在大家选购代理的时候有所帮助。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-10-04 02:31:43" itemprop="dateCreated datePublished" datetime="2018-10-04T02:31:43+08:00">2018-10-04</time>
                </span>
                <span id="/5094.html" class="post-meta-item leancloud_visitors" data-flag-title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6289.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6289.html" class="post-title-link" itemprop="url">破解网站登录加密--RSA</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="大家好，我是四毛，下面的是我的公众号，欢迎关注。"><a href="#大家好，我是四毛，下面的是我的公众号，欢迎关注。" class="headerlink" title="大家好，我是四毛，下面的是我的公众号，欢迎关注。"></a>大家好，我是四毛，下面的是我的公众号，欢迎关注。</h1>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/201802111155445124jhQyer.yasuotu.gif" alt=""> </p>
                  <h2 id="今天的内容主要讲的是破解一个网站的rsa加密，当然肯定不是破解这个算法，而是找到加密的参数，正确模拟这个算法即可。"><a href="#今天的内容主要讲的是破解一个网站的rsa加密，当然肯定不是破解这个算法，而是找到加密的参数，正确模拟这个算法即可。" class="headerlink" title="今天的内容主要讲的是破解一个网站的rsa加密，当然肯定不是破解这个算法，而是找到加密的参数，正确模拟这个算法即可。"></a><strong>今天的内容主要讲的是破解一个网站的rsa加密，当然肯定不是破解这个算法，而是找到加密的参数，正确模拟这个算法即可。</strong></h2>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2017/02/QQ图片20170205084843.jpg" alt=""></p>
                  <h2 id="1-什么是rsa算法"><a href="#1-什么是rsa算法" class="headerlink" title="1. 什么是rsa算法"></a>1. 什么是rsa算法</h2>
                  <p>下面的资料摘抄自阮一峰老师的文章， <a href="http://www.ruanyifeng.com/blog/2013/06/rsa_algorithm_part_one.html" target="_blank" rel="noopener">点这里了解更多</a> 1976年，两位美国计算机学家Whitfield Diffie 和 Martin Hellman，提出了一种崭新构思，可以在不直接传递密钥的情况下，完成解密。这被称为<a href="http://en.wikipedia.org/wiki/Diffie%E2%80%93Hellman_key_exchange" target="_blank" rel="noopener">“Diffie-Hellman密钥交换算法”</a>。这个算法启发了其他科学家。人们认识到，加密和解密可以使用不同的规则，只要这两种规则之间存在某种对应关系即可，这样就避免了直接传递密钥。 这种新的加密模式被称为”非对称加密算法”。</p>
                  <blockquote>
                    <p>（1）乙方生成两把密钥（公钥和私钥）。公钥是公开的，任何人都可以获得，私钥则是保密的。 （2）甲方获取乙方的公钥，然后用它对信息加密。 （3）乙方得到加密后的信息，用私钥解密。</p>
                  </blockquote>
                  <p>如果公钥加密的信息只有私钥解得开，那么只要私钥不泄漏，通信就是安全的。</p>
                  <h2 id="2-研究目标"><a href="#2-研究目标" class="headerlink" title="2. 研究目标"></a>2. 研究目标</h2>
                  <p>从我要研究的网站来说，就是根据参数得到正确的公钥，加密以后返回给服务器，让服务器使用私钥可以解密出正确的数据即可。 同时，本文不会将具体的网站说出来，只是给大家提供一个解决问题的思路。</p>
                  <h2 id="3-开始"><a href="#3-开始" class="headerlink" title="3. 开始"></a>3. 开始</h2>
                  <h3 id="3-1-抓包找参数"><a href="#3-1-抓包找参数" class="headerlink" title="3.1 抓包找参数"></a>3.1 抓包找参数</h3>
                  <p>首先，打开某个网站的登录页面，输入用户名，密码，验证码之类的参数， 抓包看到了下面这个页面： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/09/Selection_191.png" alt=""> 我实际输入的值全是1, 然后都被加密了， 没办法，只能去找加密的方法了。 经过一番搜索过后，才发现，原来加密的算法就在源代码里面，这里截个图： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/09/Selection_192.png" alt=""> <strong>从这里就可以看到具体的算法名以及相关的参数了，你会说，这是什么算法我都不知道啊？搜啊，用关键词搜一下就能知道了。</strong> 同时，是不是觉得这个网站好傻逼，这不太简单了吗？ 肯定不是！！！ 这么简单，说明此处也是必有玄机！！！ 至于什么玄机，到后面说，都是泪。 </p>
                  <h3 id="3-2-分析加密流程"><a href="#3-2-分析加密流程" class="headerlink" title="3.2 分析加密流程"></a>3.2 分析加密流程</h3>
                  <p>首先， 我们知道了公钥以后，解析这个公钥，就可以得到相关的参数，给大家找了示例代码</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># /usr/bin/python</span></span><br><span class="line"><span class="comment"># encoding: utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="attr">import</span> <span class="string">base64</span></span><br><span class="line"></span><br><span class="line"><span class="attr">def</span> <span class="string">str2key(s):</span></span><br><span class="line"><span class="comment">    # 对字符串解码</span></span><br><span class="line">    <span class="attr">b_str</span> = <span class="string">base64.b64decode(s)</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">if</span> <span class="string">len(b_str) &lt; 162:</span></span><br><span class="line">        <span class="attr">return</span> <span class="string">False</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">hex_str</span> = <span class="string">''</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # 按位转换成16进制</span></span><br><span class="line">    <span class="attr">for</span> <span class="string">x in b_str:</span></span><br><span class="line">        <span class="attr">h</span> = <span class="string">hex(ord(x))[2:]</span></span><br><span class="line">        <span class="attr">h</span> = <span class="string">h.rjust(2, '0')</span></span><br><span class="line">        <span class="attr">hex_str</span> <span class="string">+= h</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # 找到模数和指数的开头结束位置</span></span><br><span class="line">    <span class="attr">m_start</span> = <span class="string">29 * 2</span></span><br><span class="line">    <span class="attr">e_start</span> = <span class="string">159 * 2</span></span><br><span class="line">    <span class="attr">m_len</span> = <span class="string">128 * 2</span></span><br><span class="line">    <span class="attr">e_len</span> = <span class="string">3 * 2</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">modulus</span> = <span class="string">hex_str[m_start:m_start + m_len]</span></span><br><span class="line">    <span class="attr">exponent</span> = <span class="string">hex_str[e_start:e_start + e_len]</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">return</span> <span class="string">modulus,exponent</span></span><br><span class="line"></span><br><span class="line"><span class="attr">if</span> <span class="string">__name__ == "__main__":</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">pubkey</span> = <span class="string">"MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDC7kw8r6tq43pwApYvkJ5laljaN9BZb21TAIfT/vexbobzH7Q8SUdP5uDPXEBKzOjx2L28y7Xs1d9v3tdPfKI2LR7PAzWBmDMn8riHrDDNpUpJnlAGUqJG9ooPn8j7YNpcxCa1iybOlc2kEhmJn5uwoanQq+CA6agNkqly2H4j6wIDAQAB"</span></span><br><span class="line">    <span class="attr">key</span> = <span class="string">str2key(pubkey)</span></span><br><span class="line">    <span class="attr">print</span> <span class="string">key</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>相应的输出</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">('c2ee4c3cafab6ae37a<span class="number">700296</span>2f909e656a58da37d<span class="number">0596</span>f6d<span class="number">530087</span>d3fef7b16e86f31fb43c<span class="number">4947</span>4fe6e0cf5c404acce8f1d8bdbccbb5ecd5df6fded74f7ca<span class="number">2362</span>d1ecf<span class="number">033581983327</span>f2b887ac30cda54a499e<span class="number">500652</span>a246f68a0f9fc8fb60da5cc426b58b26ce95cda<span class="number">41219899</span>f9bb0a1a9d0abe080e9a80d92a972d87e23eb', </span><br><span class="line">'<span class="number">010001</span>')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>从代码中可以看出，解析了公钥之后得到了两个值，一个就是010001，和我们在网站源代码里面找到的值是一样的。所以，源代码里面的参数我们应该就是可以直接使用的，是不是有种找到组织的赶脚。 接下来，利用下面的代码，来对数据进行加密</p>
                  <figure class="highlight nix">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">import</span> rsa</span><br><span class="line"><span class="built_in">import</span> binascii</span><br><span class="line">def en_test():</span><br><span class="line">    <span class="attr">param_1</span> = <span class="string">"010001"</span></span><br><span class="line">    <span class="comment"># 某次我找到的</span></span><br><span class="line">    <span class="attr">param_2</span> = <span class="string">"955120AB9334B7CD52FCDB422DBF564AFD46DEBDC706F33502BBFAD9DD216A22E4D5012CB70F28473B46FB7190D08C31B4B8E76B5112ACE1C5552408961530B1C932DEEA8FC38A9A624AD22073F56F02BF453DD2C1FEA0164106D6B099CC9E5EC88C356FC164FCA47C766DD565D3D11048D27F2DD4221A0B26AB59BD7D09841F"</span></span><br><span class="line">    <span class="attr">message</span> = 'nihao'</span><br><span class="line">    <span class="attr">modulus</span> = int(param_2, <span class="number">16</span>)</span><br><span class="line">    <span class="attr">exponent</span> = int(param_1, <span class="number">16</span>)</span><br><span class="line">    <span class="attr">rsa_pubkey</span> = rsa.PublicKey(modulus, exponent)</span><br><span class="line">    <span class="attr">crypto</span> = rsa.encrypt(message, rsa_pubkey)</span><br><span class="line">    <span class="attr">data</span> = binascii.b2a_hex(crypto)</span><br><span class="line"></span><br><span class="line">    print data</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> <span class="attr">__name__</span> == '__main__':</span><br><span class="line">    en_test()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但是当我这样做完，进行模拟登录，还以为自己很牛逼的时候，服务器却给我返回了这样的结果， 目瞪狗呆啊：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"Status"</span>:<span class="literal">false</span>,<span class="attr">"ResultValue"</span>:<span class="string">""</span>,<span class="attr">"StatusCode"</span>:<span class="string">"REFRESH"</span>,<span class="attr">"StatusMessage"</span>:<span class="string">"请尝试重新登录"</span>,<span class="attr">"RecordCount"</span>:<span class="number">0</span>,<span class="attr">"Data"</span>:<span class="literal">null</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/00602RHagw1f7yyuxxeucj305i05i744.jpg" alt=""> 可以看到信息提示要刷新，但是当时是百思不得其解，为毛线要刷新？ 困惑了一会之后，我再次从头走了一遍流程，这下我才发现，原来源代码里面的那个长长的数据是会改变的，直到这个时候，我才意识到为什么要我刷新。。。。。。 服务器啊，你就不能直接说参数错误吗？刷新你大爷啊。 果然，我还是太年轻啊。</p>
                  <h1 id="果然，天上掉下的绝不是馅饼，绝逼是个陷阱。"><a href="#果然，天上掉下的绝不是馅饼，绝逼是个陷阱。" class="headerlink" title="果然，天上掉下的绝不是馅饼，绝逼是个陷阱。"></a>果然，天上掉下的绝不是馅饼，绝逼是个陷阱。</h1>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/03/你特么在逗我？.jpeg" alt=""> 知道这个坑以后就好办了，用个正则匹配一下就行了，而结果也是对的：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"Status"</span>:<span class="literal">true</span>,<span class="attr">"ResultValue"</span>:<span class="string">""</span>,<span class="attr">"StatusCode"</span>:<span class="string">"OK"</span>,<span class="attr">"StatusMessage"</span>:<span class="string">"成功"</span>,<span class="attr">"RecordCount"</span>:<span class="number">0</span>,<span class="attr">"Data"</span>:&#123;<span class="attr">"LoginUrl"</span>:<span class="string">"/System/Welcome"</span>&#125;&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="4-总结"><a href="#4-总结" class="headerlink" title="4 总结"></a>4 总结</h4>
                  <p>到这里这篇文章就结束了，这个案例相对于来说很简单，而且为了保护网站的隐私，所以没办法展开说。 有些网站的加密方式是很变态的，比如网易云音乐，知晓常见的加密方法，就可以处理大部分的情况了。 其实，网易云音乐并不是一定要加密， 有想知道非加密的方法的，可以关注我，私聊我。有点敏感，就不写文章了。 反正，我爬了1000W+的网易云音乐都是不加密的～～ 如果你有类似的问题待解决或者想了解的更清楚的细节的，欢迎关注我的公众号以后，后台私我一下。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/201802111155445124jhQyer.yasuotu.gif" alt=""></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-09-14 13:43:33" itemprop="dateCreated datePublished" datetime="2018-09-14T13:43:33+08:00">2018-09-14</time>
                </span>
                <span id="/6289.html" class="post-meta-item leancloud_visitors" data-flag-title="破解网站登录加密--RSA" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6284.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Linux <i class="label-arrow"></i>
                  </a>
                  <a href="/6284.html" class="post-title-link" itemprop="url">详解 Linux 下的用户管理、用户组管理和权限管理</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>最近和几个朋友开发项目，期间使用了一台服务器跑模型，这台服务器是多人公用的，很多人都在上面有自己的账号，互不干涉内政，一切看起来十分井然有序。近期，这个服务器上刚挂载了一块新硬盘，是一位朋友使用 root 账号挂载的，然后将磁盘映射到某个文件夹下。然而挂载好了之后发现使用普通账号没有权限在文件夹下操作，无法创建文件，于是他干脆就直接把文件夹权限改成 777 了。我心想，这还了得，改成 777 了，其他人在里面乱改咋办？会出人命的！所以，我就这件事详细梳理了一下 Linux 下的用户、用户组、文件权限等基本知识，看完这些，以后不要动不动就把文件夹改成 777 权限了。</p>
                  <h2 id="基本操作"><a href="#基本操作" class="headerlink" title="基本操作"></a>基本操作</h2>
                  <p>首选我们梳理一下 Linux 下的用户、用户组、文件权限等基本知识，然后后面通过一个案例来实际演示一下权限设置的一些操作。 首先 Linux 系统中，是有用户和用户组的概念的，用户就是身份的象征，我们必须以某一个用户身份来操作一个系统，实际上这就对应着我们登录系统时的账号。而用户组就是一些用户的集合，我们可以通过用户组来划分和统一管理某些用户。 比如我要在微信发一条朋友圈，我只想给我的亲人们看，难道我发的时候还要一个个去勾选所有的人？这未免太麻烦了。为了解决这问题，微信里面就有了标签的概念，我们可以提前给好友以标签的方式分类，发的时候直接勾选某个标签就好了，简单高效。实际上这就是用户组的概念，我们可以将某些人进行分组和归类，到时候只需要指定类别或组别就可以了，而不用一个个人去对号入座，从而节省了大量时间。 在 Linux 中，一个用户是可以属于多个组的，一个组也是可以包含多个用户的，下面我以一台 Ubuntu Linux 为例来演示一下相关的命令和操作。</p>
                  <h3 id="用户和用户组"><a href="#用户和用户组" class="headerlink" title="用户和用户组"></a>用户和用户组</h3>
                  <p>首先查看所有用户，命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cut -d<span class="string">':'</span> -f <span class="number">1</span> <span class="regexp">/etc/</span>passwd</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">root</span></span><br><span class="line"><span class="symbol">daemon</span></span><br><span class="line"><span class="keyword">bin</span></span><br><span class="line"><span class="keyword">sys</span></span><br><span class="line"><span class="keyword">...</span></span><br><span class="line"><span class="keyword">ubuntu</span></span><br><span class="line"><span class="keyword">mysql</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里一行就是一个用户名，由于太多，部分就省略了，实际上这个命令就是从密码文件中把用户名单独列出来了。 然后查看所有用户组，命令也是类似的：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cut -d<span class="string">':'</span> -f <span class="number">1</span> <span class="regexp">/etc/g</span>roup</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">root</span></span><br><span class="line"><span class="symbol">daemon</span></span><br><span class="line"><span class="keyword">bin</span></span><br><span class="line"><span class="keyword">sys</span></span><br><span class="line"><span class="keyword">...</span></span><br><span class="line"><span class="keyword">ubuntu</span></span><br><span class="line"><span class="keyword">mysql</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果基本是类似的，因为每个用户在被创建的时候都会自动创建一个同名的组作为其默认的用户组。 这里我是使用 ubuntu 这个账号来登录的，下面我来看下 ubuntu 这个账号是属于哪些组。 查看一个用户所属组的命令格式如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">gorups <span class="tag">&lt;<span class="name">username</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就是 groups 命令加上用户名就能查看该用户名所属的组了，如果不加用户名的话就默认是当前用户。 例如查看 ubuntu 这个用户所属于的组，命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">groups ubuntu</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ubuntu : <span class="type">ubuntu</span> adm cdrom sudo dip plugdev lxd lpadmin sambashare</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>还不少，这个用户被分配到了很多组下，比如同名的组 ubuntu，还有 sudo 组，另外还有一些其他的组。 其中 sudo 组比较特殊，如果被分到了这个组里面就代表该账号拥有 root 权限，可以使用 sudo 命令。 了解了怎样查看用户所属的组，我们也应该反过来了解如何查看一个用户组里面包含哪些用户啊。 查看某个用户组下所有用户命令如下：</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">members</span> &lt;<span class="built_in">group</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不过这个命令不是自带的，需要额外安装 members 包，命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> install members</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如查看 sudo 用户组下的所有用户，即拥有 root 权限的用户：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">members sudo</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ubuntu hadoop</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到拥有 root 权限的用户有两个，ubuntu 和 hadoop，当然不同的机器结果肯定是不一样的。 接下来介绍一个比较有用的命令，就是 id 命令，它可以用来查看用户的所属组别，格式如下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> &lt;username&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如查看 ubuntu 用户的信息，就是这样：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> ubuntu</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">uid=<span class="number">500</span>(ubuntu) gid=<span class="number">500</span>(ubuntu) groups=<span class="number">500</span>(ubuntu),<span class="number">4</span>(adm),<span class="number">24</span>(cdrom),<span class="number">27</span>(sudo),<span class="number">30</span>(dip),<span class="number">46</span>(plugdev),<span class="number">110</span>(lxd),<span class="number">115</span>(lpadmin),<span class="number">116</span>(sambashare)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里有一个 gid，作为主工作组，后面还有个 groups，它列出了用户所在的所有组。主工作组只有一个，而后者的数量则不限。可以看到用户组的结果和使用 groups 命令看到的结果是一致的。 接下来我们再来了解一下如何创建一个用户和怎样为用户分配组别。 添加一个用户命令格式如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>&lt;username&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如我要添加一个用户 cqc，命令就可以这么写：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>cqc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用的命令前面都带有 sudo，因为毕竟是系统级别的操作。 添加一个组的命令格式如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo groupadd <span class="tag">&lt;<span class="name">group</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>格式是类似的，后面跟一个组的名称就可以了，例如我要为我的实验室创建一个用户组，那么就可以使用如下命令：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo groupadd lab</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>创建完了用户和组，那得把它们关联起来吧，关联的意思就是把某个用户加入到某个组里面，命令格式如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo adduser <span class="tag">&lt;<span class="name">username</span>&gt;</span> <span class="tag">&lt;<span class="name">group</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者使用 usermod 命令：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo usermod -G <span class="tag">&lt;<span class="name">group</span>&gt;</span> <span class="tag">&lt;<span class="name">username</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果要添加多个组的话，可以通过 -a 选项指定多个名称：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo usermod -aG &lt;group1,group2,group3..&gt; &lt;username&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如我要将 cqc 用户添加到 sudo 用户组中，命令就是：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>cqc sudo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo usermod -G sudo cqc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就为用户和用户组做好关联了。</p>
                  <h3 id="文件权限管理"><a href="#文件权限管理" class="headerlink" title="文件权限管理"></a>文件权限管理</h3>
                  <p>了解了这些之后，我们再来了解一下文件权限的相关知识，下面我们先随便找一个目录，查看一下文件的列表。 列出某个目录下文件详细信息的命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ll</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者使用：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ls -l</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如我这里列出了 /etc/nginx 目录下的文件列表：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">total <span class="number">80</span></span><br><span class="line">drwxr-xr-x   <span class="number">7</span> root root  <span class="number">4096</span> Jun <span class="number">21</span> <span class="number">22</span>:<span class="number">16</span> ./</span><br><span class="line">drwxr-xr-x <span class="number">103</span> root root  <span class="number">4096</span> Sep  <span class="number">4</span> <span class="number">18</span>:<span class="number">04</span> ../</span><br><span class="line">drwxr-xr-x   <span class="number">2</span> root root  <span class="number">4096</span> Jul <span class="number">12</span>  <span class="number">2017</span> conf.d/</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">1077</span> Feb <span class="number">12</span>  <span class="number">2017</span> fastcgi.conf</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">1007</span> Feb <span class="number">12</span>  <span class="number">2017</span> fastcgi_params</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">2837</span> Feb <span class="number">12</span>  <span class="number">2017</span> koi-utf</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">2223</span> Feb <span class="number">12</span>  <span class="number">2017</span> koi-win</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">3957</span> Feb <span class="number">12</span>  <span class="number">2017</span> mime.types</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">1505</span> Jun <span class="number">21</span> <span class="number">20</span>:<span class="number">24</span> nginx.conf</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root <span class="number">12288</span> Jun <span class="number">21</span> <span class="number">20</span>:<span class="number">44</span> .nginx.conf.swp</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root   <span class="number">180</span> Feb <span class="number">12</span>  <span class="number">2017</span> proxy_params</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root   <span class="number">636</span> Feb <span class="number">12</span>  <span class="number">2017</span> scgi_params</span><br><span class="line">drwxr-xr-x   <span class="number">2</span> root root  <span class="number">4096</span> Jun <span class="number">21</span> <span class="number">22</span>:<span class="number">42</span> sites-available/</span><br><span class="line">drwxr-xr-x   <span class="number">2</span> root root  <span class="number">4096</span> Jun <span class="number">21</span> <span class="number">19</span>:<span class="number">08</span> sites-enabled/</span><br><span class="line">drwxr-xr-x   <span class="number">2</span> root root  <span class="number">4096</span> Jun <span class="number">21</span> <span class="number">19</span>:<span class="number">08</span> snippets/</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root   <span class="number">664</span> Feb <span class="number">12</span>  <span class="number">2017</span> uwsgi_params</span><br><span class="line">drwxr-xr-x   <span class="number">2</span> root root  <span class="number">4096</span> Jun <span class="number">22</span> <span class="number">02</span>:<span class="number">44</span> vhosts/</span><br><span class="line">-rw-r--r--   <span class="number">1</span> root root  <span class="number">3071</span> Feb <span class="number">12</span>  <span class="number">2017</span> win-utf</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们注意到了每一行都是一个文件或文件夹的信息，一共包括七列：</p>
                  <ul>
                    <li>第一列是文件的权限信息</li>
                    <li>第二列表示该文件夹连接的文件数</li>
                    <li>第三列表示文件所属用户</li>
                    <li>第四列表示文件所属用户组</li>
                    <li>第五列表示文件大小（字节）</li>
                    <li>第六列表示最后修改日期</li>
                    <li>第七列表示文件名</li>
                  </ul>
                  <p>其中第一列的文件权限信息是非常重要的，它由十个字符组成：</p>
                  <ul>
                    <li>第一个字符代表文件的类型，有三种，- 代表这是一个文件，d 代表这是一个文件夹，l 代表这是一个链接。</li>
                    <li>第 2-4 个字符代表文件所有者对该文件的权限，r 就是读，w 就是写，x 就是执行，如果是文件夹的话，执行就意味着查看文件夹下的内容，例如 rw- 就代表文件所有者可以对该文件进行读取和写入。</li>
                    <li>第 5-7 个字符代表文件所属组对该文件的权限，含义是一样的，如 r-x 就代表该文件所属组内的所有用户对该文件有读取和执行的权限。</li>
                    <li>第 8-10 个字符代表是其他用户对该文件的权限，含义也是一样的，如 r— 就代表非所有者，非用户组的用户只拥有对该文件的读取权限。</li>
                  </ul>
                  <p>我们可以使用 chmod 命令来改变文件或目录的权限，有这么几种用法。 一种是数字权限命名，rwx 对应一个二进制数字，如 101 就代表拥有读取和执行的权限，而转为十进制的话，r 就代表 4，w 就代表 2，x 就代表 1，然后三个数字加起来就和二进制数字对应起来了。如 7=4+2+1，这就对应着 rwx；5=4+1，这就对应着 r-x。所以，相应地 777 就代表了 rwxrwxrwx，即所有者、所属用户组、其他用户对该文件都拥有读取、写入、执行的权限，这是相当危险的！ 赋予权限的命令如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod <span class="tag">&lt;<span class="name">permission</span>&gt;</span> <span class="tag">&lt;<span class="name">file</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如我要为一个 file.txt 赋予 777 权限，就写成：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod <span class="number">777</span> file.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外我们也可以使用代号来赋予权限，代号有 u、g、o、a 四中，分别代表所有者权限，用户组权限，其他用户权限和所有用户权限，这些代号后面通过 + 和 - 符号来控制权限的添加和移除，再后面跟上权限类型就好，例如：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod u-<span class="meta">x</span> <span class="meta">file</span>.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就是给所有者移除 x 权限，也就是执行权限。</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod g+w <span class="built_in">file</span>.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就是为用户组添加 w 权限，即写入权限。 另外如果是文件夹的话还可以对文件夹进行递归赋权限操作，如：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod -R <span class="number">777</span> share</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就是将 share 文件夹和其内所有内容都赋予 777 权限。 好，有了权限的标识，那我们还得把用户和用户组与文件关联起来啊，这里使用的命令就是 chown 和 chgrp 命令。 命令格式如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chown <span class="tag">&lt;<span class="name">username</span>&gt;</span> <span class="tag">&lt;<span class="name">file</span>&gt;</span></span><br><span class="line">sudo chgrp <span class="tag">&lt;<span class="name">group</span>&gt;</span> <span class="tag">&lt;<span class="name">file</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如我要将 file.txt 的所有者换成 cqc，那就可以使用如下命令：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chown cqc <span class="built_in">file</span>.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果我要将 file.txt 所属用户组换成 lab，那就可以使用如下命令：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chgrp <span class="keyword">lab</span> <span class="keyword">file</span>.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外同样可以使用 -R 来进行递归操作，如将 share 文件夹及其内所有内容的所有者都换成 cqc，命令如下：</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo <span class="keyword">chown</span> -R cqc share/</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好，了解了 chown、chgrp、chmod 之后，我们就可以灵活地对文件权限进行控制了。</p>
                  <h2 id="实战演示"><a href="#实战演示" class="headerlink" title="实战演示"></a>实战演示</h2>
                  <p>可能上面说起来有点抽象，下面我们以一个实例来演示一下权限控制的流程，通过这个流程，相信理解以上的命令都不在话下了。 首先情况是这样的，我要在某台主机上共享一些文件给我实验室的人，但这台主机上还有其他非实验室的人在使用，我只想让实验室的人查看和修改这些文件，其他人不行。 另外我自己的账号要有最高权限来管理这些文件的共享权限，即要有 root 权限。 现在我已经登录了一个 ubuntu 的账号，是系统初始化的，拥有 root 权限。 下面我就模拟创建三个账号和一个用户组，来得到如下效果：</p>
                  <ul>
                    <li>账号 cqc 是我自己使用的账号，拥有最高权限，可以自由调整文件权限信息，可以自由为某个用户分配用户组。</li>
                    <li>账号 lbd 是我实验室的人员，没有 root 权限，但能查看和修改我共享的文件。</li>
                    <li>账号 slb 不是我实验室的人员，没有 root 权限，也不能修改我共享的文件。</li>
                  </ul>
                  <h3 id="创建自己的账户"><a href="#创建自己的账户" class="headerlink" title="创建自己的账户"></a>创建自己的账户</h3>
                  <p>首先我先为自己创建一个账户，添加一个 cqc 的用户：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>cqc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后会提示输入密码和其他信息：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Adding<span class="built_in"> user </span>`cqc<span class="string">' ...</span></span><br><span class="line"><span class="string">Adding new group `cqc'</span> (1002) <span class="built_in">..</span>.</span><br><span class="line">Adding new<span class="built_in"> user </span>`cqc<span class="string">' (1002) with group `cqc'</span> <span class="built_in">..</span>.</span><br><span class="line">Creating home directory `/home/cqc<span class="string">' ...</span></span><br><span class="line"><span class="string">Copying files from `/etc/skel'</span> <span class="built_in">..</span>.</span><br><span class="line">Enter new UNIX password: </span><br><span class="line">Retype new UNIX password: </span><br><span class="line">passwd: password updated successfully</span><br><span class="line">Changing the<span class="built_in"> user </span>information <span class="keyword">for</span> cqc</span><br><span class="line">Enter the new value, <span class="keyword">or</span> press ENTER <span class="keyword">for</span> the default</span><br><span class="line">        Full Name []: </span><br><span class="line">        Room Number []: </span><br><span class="line">        Work Phone []: </span><br><span class="line">        Home Phone []: </span><br><span class="line">        Other []: </span><br><span class="line">Is the information correct? [Y/n]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时候发现一个同名的组就被创建了，查看下 cqc 所在的组：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">groups cqc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cqc : <span class="type">cqc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再用 id 命令查看下信息：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> cqc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">uid=1002(<span class="name">cqc</span>) gid=1002(<span class="name">cqc</span>) groups=1002(<span class="name">cqc</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到当前 cqc 只属于 cqc 用户组。 接下来我们创建一个用户组，叫做 lab，来标明我的实验室，命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo groupadd lab</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后查看下用户组里面的成员：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">members lab</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>没有任何结果，说明我们创建了一个空的组，没有任何成员。 然后我们将刚才创建的 cqc 加入到该组中，因为我自己也属于该实验室，肯定也要加进来，命令如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>cqc lab</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Adding<span class="built_in"> user </span>`cqc<span class="string">' to group `lab'</span> <span class="built_in">..</span>.</span><br><span class="line">Adding<span class="built_in"> user </span>cqc <span class="keyword">to</span><span class="built_in"> group </span>lab</span><br><span class="line">Done.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后查看下组内成员：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">members lab</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">cqc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样 lab 组内就有了 cqc 这个用户了。 别忘了 cqc 还需要拥有 root 权限，所以我们还需要将 cqc 添加到 sudo 组内，命令如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>cqc sudo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Adding<span class="built_in"> user </span>`cqc<span class="string">' to group `sudo'</span> <span class="built_in">..</span>.</span><br><span class="line">Adding<span class="built_in"> user </span>cqc <span class="keyword">to</span><span class="built_in"> group </span>sudo</span><br><span class="line">Done.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就成功加入到 sudo 组了，cqc 也就是我的账户就可以使用 sudo 命令了。 查看下用户状态：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> cqc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">uid</span>=1002(cqc) <span class="attribute">gid</span>=1002(cqc) <span class="attribute">groups</span>=1002(cqc),27(sudo),1003(lab)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样 cqc 就属于三个用户组了，既是实验室成员，又拥有 root 权限。 上面的分配用户组的命令我们也可以使用 usermod 来实现：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo</span> usermod -aG sudo,lab cqc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就添加到多个组了。</p>
                  <h3 id="添加实验室用户"><a href="#添加实验室用户" class="headerlink" title="添加实验室用户"></a>添加实验室用户</h3>
                  <p>接下来，再添加实验室的另外一个人员 lbd，然后将其添加到 lab 组中，流程是类似的，命令如下：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo <span class="keyword">adduser </span><span class="keyword">lbd</span></span><br><span class="line"><span class="keyword">sudo </span><span class="keyword">adduser </span><span class="keyword">lbd </span>lab</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行完毕之后，id 命令查看其信息：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> lbd</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">uid</span>=1004(lbd) <span class="attribute">gid</span>=1005(lbd) <span class="attribute">groups</span>=1005(lbd),1003(lab)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就成功创建 lbd，并将其添加到实验室 lab 组了。</p>
                  <h3 id="添加非实验室用户"><a href="#添加非实验室用户" class="headerlink" title="添加非实验室用户"></a>添加非实验室用户</h3>
                  <p>最后另外添加一个用户 slb，非实验室成员，只创建账户就好了，命令如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>slb</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但是我们不把他加入 lab 组中。 查看他的状态：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">id</span> slb</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">uid=1003(<span class="name">slb</span>) gid=1004(<span class="name">slb</span>) groups=1004(<span class="name">slb</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以三人的状态是这样的：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id cqc</span><br><span class="line"><span class="attribute">uid</span>=1002(cqc) <span class="attribute">gid</span>=1002(cqc) <span class="attribute">groups</span>=1002(cqc),27(sudo),1003(lab)</span><br><span class="line">id lbd</span><br><span class="line"><span class="attribute">uid</span>=1004(lbd) <span class="attribute">gid</span>=1005(lbd) <span class="attribute">groups</span>=1005(lbd),1003(lab)</span><br><span class="line">id slb</span><br><span class="line"><span class="attribute">uid</span>=1003(slb) <span class="attribute">gid</span>=1004(slb) <span class="attribute">groups</span>=1004(slb)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="文件权限分配"><a href="#文件权限分配" class="headerlink" title="文件权限分配"></a>文件权限分配</h3>
                  <p>接下来我们创建一个文件夹来共享实验室数据，放在 /srv 目录下。然后调用 mkdir 命令创建名称为 share 的文件夹，命令如下：</p>
                  <figure class="highlight dos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">cd</span> /srv</span><br><span class="line">sudo <span class="built_in">mkdir</span> share</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>注意这里我还是使用 ubuntu 账户来创建的。 先看下当前目录权限：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ls -l</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">total <span class="number">12</span></span><br><span class="line">drwxr-xr-x  <span class="number">3</span> root root <span class="number">4096</span> Sep  <span class="number">4</span> <span class="number">18</span>:<span class="number">17</span> ./</span><br><span class="line">drwxr-xr-x <span class="number">24</span> root root <span class="number">4096</span> Sep  <span class="number">4</span> <span class="number">18</span>:<span class="number">17</span> ../</span><br><span class="line">drwxr-xr-x  <span class="number">2</span> root root <span class="number">4096</span> Sep  <span class="number">4</span> <span class="number">18</span>:<span class="number">17</span> share/</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到 share 文件的所有者是 root，用户组也是 root，权限是 755，即只有 root 拥有修改权限，其他的只有读取和执行权限。 然后进入 share 文件夹创建一个 names.txt：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">cd</span> share</span><br><span class="line">sudo <span class="keyword">vi</span> names.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>编辑内容如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">cqc</span></span><br><span class="line"><span class="attribute">lbd</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>保存完毕之后，这时查看一下文件权限，如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">-rw-r----- <span class="number">1</span> root root    <span class="number">8</span> Sep  <span class="number">4</span> <span class="number">20</span>:<span class="number">00</span> names.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>权限是 640，这表明只有所有者 root 拥有写入的权限，所在组只有读的权限。 这时开启另外一个终端，登录 cqc 账号，实际上是不能查看和修改任何该文件的内容的，下面的修改和读取命令都会提示权限不够：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">vi</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br><span class="line"><span class="selector-tag">cat</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>为什么呢？因为该文件是刚才由 ubuntu 账号使用 sudo 命令创建的，因此文件的所有者是 root，并不是 cqc，因此即使文件的权限是 640，那也就不能使用文件所有者的权限，而且 cqc 也不属于 root 组，所以也不能使用文件组的权限了，因此什么都看不了，什么都改不了。 但 cqc 属于 sudo 组啊，可以利用 sudo 命令临时获取 root 权限，临时以 root 的身份来操作该文件，这样就可以来查看和修改文件了，因此下面的命令是有效的：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">vi</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">cat</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但这样还是需要使用 sudo 才能修改，很不方便。 这时如果我们把文件的所有者改成 cqc，情况那就不一样了。 使用 ubuntu 账号，对 names.txt 更改其所有者为 cqc，改的命令如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">chown</span> <span class="selector-tag">cqc</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时查看下文件信息：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">-rw-r----- <span class="number">1</span> cqc  root    <span class="number">8</span> Sep  <span class="number">4</span> <span class="number">20</span>:<span class="number">29</span> names.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到所有者信息已经变成了 cqc，这样 cqc 账号再直接查看和修改，那就可以了，不再需要 sudo 命令：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">vi</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br><span class="line"><span class="selector-tag">cat</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就不会有权限提示，当然加上 sudo 更是没问题。 好，接下来 lbd 呢？我们登录试试修改。 首先当前的文件状态是这样的：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">-rw-r----- <span class="number">1</span> cqc  root    <span class="number">8</span> Sep  <span class="number">4</span> <span class="number">20</span>:<span class="number">31</span> names.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>lbd 不是所有者了，因此前面的 rw- 权限是没什么用的，但他属于 lab 组，而该文件对于用户组的权限是 r—，也就是读取权限。 我们使用 lbd 账号来尝试看下文件的内容：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">cat</span> names.txt </span><br><span class="line"><span class="keyword">ca</span><span class="variable">t:</span> names.tx<span class="variable">t:</span> Permission denied</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>很遗憾，又没有权限。因为什么？因为这个文件的用户组并不是 lab 啊，而 lbd 这个用户又不属于 root 组，所以没有任何权限。 那咋办？将文件的用户组改成 lab 就好了，使用 ubuntu 账号或 cqc 账号来操作：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">chgrp</span> <span class="selector-tag">lab</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就成功将文件所属用户组改成 lab 了，接下来再使用 lbd 账号查看下文件内容：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">cat</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就成功读取了。 然而 lbd 现在是没有写入权限的，因为对于用户组来说，该文件的权限是 r—，如果要获取写入权限，我们可以使用如下命令：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">chmod</span> <span class="selector-tag">g</span>+<span class="selector-tag">w</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod <span class="number">660</span> names.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就相当于赋予了 rw- 权限，下面我们再使用 lbd 账号尝试修改这个文件：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">vi</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就没问题了。 那么对于非实验室同学 slb 呢？它没有任何权限，我们登录 slb 账号尝试修改和读取该文件：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">cat</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br><span class="line"><span class="selector-tag">vi</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>均无权限。 所以说，这样我们就成功为实验室的人员赋予了权限，而非实验室的人则没有任何权限。 如果我要为 slb 赋予读取权限咋办呢？很简单，添加一下就好了：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">chmod</span> <span class="selector-tag">o</span>+<span class="selector-tag">r</span> <span class="selector-tag">names</span><span class="selector-class">.txt</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就是为其他用户添加了读取权限。这时 slb 就可以读取文件，但不能修改文件，也是比较安全的。 好，如果我的文件非常多呢？比如十几二十个，都放在 share 文件夹内，那不能一个个进行权限设置吧？ 这时候我们只需要针对文件夹进行操作即可，下面的命令就可以为 share 文件夹赋予 775 权限，即所有者 cqc 和所在组 lab 可对其进行查看和修改，其他的人只能看不能改：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo chmod -R <span class="number">775</span> <span class="keyword">share</span>/</span><br><span class="line">sudo chown -R cqc <span class="keyword">share</span>/</span><br><span class="line">sudo chgrp -R lab <span class="keyword">share</span>/</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>注意文件夹一般都会赋予 x 权限，不然连进入文件夹的权限都没有。这也就是文件夹一般会赋予 775、755，而文件会赋予 664、600、644、640 的原因了。 赋予 775 权限之后，share 的权限就变成了：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">drwxrwxr-x  <span class="number">2</span> cqc  lab  <span class="number">4096</span> Sep  <span class="number">4</span> <span class="number">20</span>:<span class="number">31</span> share/</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样其他用户就只能看，不能改，这样普通文件就没什么问题了。 如文件夹内包含了可执行文件，还可以单独为其他用户针对可执行文件去除 x 权限，如去除 Python 文件的可执行权限：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo</span> chmod o-x <span class="regexp">*.py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好了，到现在为止，我们就得心应手地完成了权限控制了！ 相信如果你耐心看完的话，什么用户管理、权限管理，都不在话下！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-09-04 23:20:13" itemprop="dateCreated datePublished" datetime="2018-09-04T23:20:13+08:00">2018-09-04</time>
                </span>
                <span id="/6284.html" class="post-meta-item leancloud_visitors" data-flag-title="详解 Linux 下的用户管理、用户组管理和权限管理" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6281.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Linux <i class="label-arrow"></i>
                  </a>
                  <a href="/6281.html" class="post-title-link" itemprop="url">如何给Azure云服务器扩展云磁盘</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本文介绍一下如何给 Azure 的云服务器增加一块磁盘。</p>
                  <h2 id="页面操作"><a href="#页面操作" class="headerlink" title="页面操作"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Azure%E3%80%91%E7%BB%99Azure%E6%9C%8D%E5%8A%A1%E5%99%A8%E6%89%A9%E5%B1%95%E7%A3%81%E7%9B%98.md#%E9%A1%B5%E9%9D%A2%E6%93%8D%E4%BD%9C" target="_blank" rel="noopener"></a>页面操作</h2>
                  <p>首先切换到磁盘页面，然后点击添加数据磁盘按钮： <a href="https://github.com/Germey/AI/blob/master/assets/2018-08-02-16-42-27.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-08-02-16-42-27.jpg" alt=""></a> 然后选定存储容器，这里使用的是存储账户 Blob，然后点击确定按钮： <a href="https://github.com/Germey/AI/blob/master/assets/2018-08-02-16-43-38.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-08-02-16-43-38.jpg" alt=""></a> 主机缓存切换为“读/写”，然后点击保存： <a href="https://github.com/Germey/AI/blob/master/assets/2018-08-02-16-44-09.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-08-02-16-44-09.jpg" alt=""></a> 这样就添加好了。</p>
                  <h2 id="挂载磁盘"><a href="#挂载磁盘" class="headerlink" title="挂载磁盘"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Azure%E3%80%91%E7%BB%99Azure%E6%9C%8D%E5%8A%A1%E5%99%A8%E6%89%A9%E5%B1%95%E7%A3%81%E7%9B%98.md#%E6%8C%82%E8%BD%BD%E7%A3%81%E7%9B%98" target="_blank" rel="noopener"></a>挂载磁盘</h2>
                  <p>接下来回到 Linux 服务器下，我们需要将磁盘进行挂载。 首先 SSH 连接到服务器，然后使用 dmesg 命令来查找磁盘：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">dmesg <span class="string">| grep SCSI</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出类似如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-attr">[    0.728389]</span> <span class="selector-tag">SCSI</span> <span class="selector-tag">subsystem</span> <span class="selector-tag">initialized</span></span><br><span class="line"><span class="selector-attr">[    2.139341]</span> <span class="selector-tag">Block</span> <span class="selector-tag">layer</span> <span class="selector-tag">SCSI</span> <span class="selector-tag">generic</span> (<span class="selector-tag">bsg</span>) <span class="selector-tag">driver</span> <span class="selector-tag">version</span> 0<span class="selector-class">.4</span> <span class="selector-tag">loaded</span> (<span class="selector-tag">major</span> 244)</span><br><span class="line"><span class="selector-attr">[    2.978928]</span> <span class="selector-tag">sd</span> 1<span class="selector-pseudo">:0</span><span class="selector-pseudo">:1</span><span class="selector-pseudo">:0</span>: <span class="selector-attr">[sdb]</span> <span class="selector-tag">Attached</span> <span class="selector-tag">SCSI</span> <span class="selector-tag">disk</span></span><br><span class="line"><span class="selector-attr">[    3.341183]</span> <span class="selector-tag">sd</span> 0<span class="selector-pseudo">:0</span><span class="selector-pseudo">:0</span><span class="selector-pseudo">:0</span>: <span class="selector-attr">[sda]</span> <span class="selector-tag">Attached</span> <span class="selector-tag">SCSI</span> <span class="selector-tag">disk</span></span><br><span class="line"><span class="selector-attr">[   18.397942]</span> <span class="selector-tag">Loading</span> <span class="selector-tag">iSCSI</span> <span class="selector-tag">transport</span> <span class="selector-tag">class</span> <span class="selector-tag">v2</span><span class="selector-class">.0-870</span>.</span><br><span class="line"><span class="selector-attr">[ 6641.364794]</span> <span class="selector-tag">sd</span> 3<span class="selector-pseudo">:0</span><span class="selector-pseudo">:0</span><span class="selector-pseudo">:0</span>: <span class="selector-attr">[sdc]</span> <span class="selector-tag">Attached</span> <span class="selector-tag">SCSI</span> <span class="selector-tag">disk</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 sdc 就是我们新添加的一块硬盘。 然后我们使用 fdisk 对其进行分区，将其设置为分区 1 中的主磁盘，并接受其他的默认值，命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo fdisk <span class="regexp">/dev/</span>sdc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>使用 n 命令添加新分区，然后 p 选择主分区，其他的默认：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Welcome <span class="keyword">to</span> fdisk (util-linux 2.27.1).</span><br><span class="line">Changes will remain <span class="keyword">in</span> memory only, until you decide <span class="keyword">to</span> write them.</span><br><span class="line">Be careful before using the write command.</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Device does <span class="keyword">not</span> contain a recognized partition table.</span><br><span class="line">Created a new DOS disklabel with disk identifier 0xc305fe54.</span><br><span class="line"></span><br><span class="line">Command (m <span class="keyword">for</span> help): n</span><br><span class="line">Partition type</span><br><span class="line">   p   primary (0 primary, 0 extended, 4 free)</span><br><span class="line">   e   extended (container <span class="keyword">for</span> logical partitions)</span><br><span class="line">Select (default p): p</span><br><span class="line">Partition number (1-4,<span class="built_in"> default </span>1): </span><br><span class="line">First sector (2048-2145386495,<span class="built_in"> default </span>2048): </span><br><span class="line">Last sector, +sectors <span class="keyword">or</span> +size&#123;K,M,G,T,P&#125; (2048-2145386495,<span class="built_in"> default </span>2145386495): </span><br><span class="line"></span><br><span class="line">Created a new partition 1 of<span class="built_in"> type </span><span class="string">'Linux'</span> <span class="keyword">and</span> of size 1023 GiB.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后使用 p 打印分区表并使用 w 将表写入磁盘，然后退出：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Command (m for <span class="keyword">help</span>): p</span><br><span class="line">Disk /dev/sdc: <span class="number">1023</span> GiB, <span class="number">1098437885952</span> <span class="keyword">bytes</span>, <span class="number">2145386496</span> sectors</span><br><span class="line">Units: sectors <span class="keyword">of</span> <span class="number">1</span> * <span class="number">512</span> = <span class="number">512</span> <span class="keyword">bytes</span></span><br><span class="line">Sector <span class="keyword">size</span> (<span class="keyword">logical</span>/<span class="keyword">physical</span>): <span class="number">512</span> <span class="keyword">bytes</span> / <span class="number">512</span> <span class="keyword">bytes</span></span><br><span class="line">I/O <span class="keyword">size</span> (<span class="keyword">minimum</span>/<span class="keyword">optimal</span>): <span class="number">512</span> <span class="keyword">bytes</span> / <span class="number">512</span> <span class="keyword">bytes</span></span><br><span class="line">Disklabel <span class="keyword">type</span>: dos</span><br><span class="line">Disk identifier: <span class="number">0xc305fe54</span></span><br><span class="line"></span><br><span class="line">Device     Boot <span class="keyword">Start</span>        <span class="keyword">End</span>    Sectors  <span class="keyword">Size</span> <span class="keyword">Id</span> <span class="keyword">Type</span></span><br><span class="line">/dev/sdc1        <span class="number">2048</span> <span class="number">2145386495</span> <span class="number">2145384448</span> <span class="number">1023</span>G <span class="number">83</span> Linux</span><br><span class="line"></span><br><span class="line">Command (m <span class="keyword">for</span> <span class="keyword">help</span>): w</span><br><span class="line">The <span class="keyword">partition</span> <span class="keyword">table</span> has been altered.</span><br><span class="line"><span class="keyword">Calling</span> ioctl() <span class="keyword">to</span> re-<span class="keyword">read</span> <span class="keyword">partition</span> table.</span><br><span class="line">Syncing disks.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来使用 mkfs 命令将文件系统写入分区，指定文件系统的类型和设备名称：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo mkfs -t ext4 <span class="regexp">/dev/</span>sdc1</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出类似如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">mke2fs <span class="number">1.42</span><span class="number">.13</span> (<span class="number">17</span>-May<span class="number">-2015</span>)</span><br><span class="line">Creating filesystem with <span class="number">268173056</span> <span class="number">4</span>k blocks <span class="keyword">and</span> <span class="number">67043328</span> inodes</span><br><span class="line">Filesystem UUID: d744c5d7-f4d1<span class="number">-4f</span>81<span class="number">-9f</span>56<span class="number">-59</span>dfab956782</span><br><span class="line">Superblock backups stored on blocks: </span><br><span class="line">        <span class="number">32768</span>, <span class="number">98304</span>, <span class="number">163840</span>, <span class="number">229376</span>, <span class="number">294912</span>, <span class="number">819200</span>, <span class="number">884736</span>, <span class="number">1605632</span>, <span class="number">2654208</span>, </span><br><span class="line">        <span class="number">4096000</span>, <span class="number">7962624</span>, <span class="number">11239424</span>, <span class="number">20480000</span>, <span class="number">23887872</span>, <span class="number">71663616</span>, <span class="number">78675968</span>, </span><br><span class="line">        <span class="number">102400000</span>, <span class="number">214990848</span></span><br><span class="line"></span><br><span class="line">Allocating group tables: done                            </span><br><span class="line">Writing inode tables: done                            </span><br><span class="line">Creating journal (<span class="number">32768</span> blocks): done</span><br><span class="line">Writing superblocks <span class="keyword">and</span> filesystem accounting information: done</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后使用 mkdir 创建一个目录来装载该文件系统，然后挂载：</p>
                  <figure class="highlight jboss-cli">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo mkdir <span class="string">/datadrive</span></span><br><span class="line">sudo mount <span class="string">/dev/sdc1</span> <span class="string">/datadrive</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就挂载成功了。</p>
                  <h2 id="添加引导信息"><a href="#添加引导信息" class="headerlink" title="添加引导信息"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Azure%E3%80%91%E7%BB%99Azure%E6%9C%8D%E5%8A%A1%E5%99%A8%E6%89%A9%E5%B1%95%E7%A3%81%E7%9B%98.md#%E6%B7%BB%E5%8A%A0%E5%BC%95%E5%AF%BC%E4%BF%A1%E6%81%AF" target="_blank" rel="noopener"></a>添加引导信息</h2>
                  <p>若要确保在重新引导后自动重新装载驱动器，必须将其添加到 /etc/fstab 文件。 此外，强烈建议在 /etc/fstab 中使用 UUID（全局唯一标识符）来引用驱动器而不是只使用设备名称（例如 /dev/sdc1）。 如果 OS 在启动过程中检测到磁盘错误，使用 UUID 可以避免将错误的磁盘装载到给定位置。 然后，为剩余的数据磁盘分配这些设备 ID。 若要查找新驱动器的 UUID，请使用 blkid 实用工具：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo -i blkid</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输入类似如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">/dev/sdb1: <span class="attribute">UUID</span>=<span class="string">"d5b61f40-4129-4b39-b861-c2d3b09cee69"</span> <span class="attribute">TYPE</span>=<span class="string">"ext4"</span> <span class="attribute">PARTUUID</span>=<span class="string">"4927b944-01"</span></span><br><span class="line">/dev/sda1: <span class="attribute">LABEL</span>=<span class="string">"cloudimg-rootfs"</span> <span class="attribute">UUID</span>=<span class="string">"b2e62f4f-d338-470e-9ae7-4fc0e014858c"</span> <span class="attribute">TYPE</span>=<span class="string">"ext4"</span> <span class="attribute">PARTUUID</span>=<span class="string">"577c3e7c-01"</span></span><br><span class="line">/dev/sdc1: <span class="attribute">UUID</span>=<span class="string">"d744c5d7-f4d1-4f81-9f56-59dfab956782"</span> <span class="attribute">TYPE</span>=<span class="string">"ext4"</span> <span class="attribute">PARTUUID</span>=<span class="string">"c305fe54-01"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后编辑 /etc/fstab，添加下面一行：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">UUID=d744c5d7-f4d1<span class="number">-4f</span>81<span class="number">-9f</span>56<span class="number">-59</span>dfab956782       /datadrive      ext4    defaults,nofail <span class="number">1</span>      <span class="number">2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后保存退出即可。 这样就成功添加了一块外部磁盘。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-09-04 15:42:35" itemprop="dateCreated datePublished" datetime="2018-09-04T15:42:35+08:00">2018-09-04</time>
                </span>
                <span id="/6281.html" class="post-meta-item leancloud_visitors" data-flag-title="如何给Azure云服务器扩展云磁盘" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6255.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Other <i class="label-arrow"></i>
                  </a>
                  <a href="/6255.html" class="post-title-link" itemprop="url">Ubuntu 搭建 Elasticsearch 6 集群流程</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="为何要搭建-Elasticsearch-集群"><a href="#为何要搭建-Elasticsearch-集群" class="headerlink" title="为何要搭建 Elasticsearch 集群"></a>为何要搭建 Elasticsearch 集群</h2>
                  <p>凡事都要讲究个为什么。在搭建集群之前，我们首先先问一句，为什么我们需要搭建集群？它有什么优势呢？</p>
                  <h3 id="高可用性"><a href="#高可用性" class="headerlink" title="高可用性"></a>高可用性</h3>
                  <p>Elasticsearch 作为一个搜索引擎，我们对它的基本要求就是存储海量数据并且可以在非常短的时间内查询到我们想要的信息。所以第一步我们需要保证的就是 Elasticsearch 的高可用性，什么是高可用性呢？它通常是指，通过设计减少系统不能提供服务的时间。假设系统一直能够提供服务，我们说系统的可用性是 100%。如果系统在某个时刻宕掉了，比如某个网站在某个时间挂掉了，那么就可以它临时是不可用的。所以，为了保证 Elasticsearch 的高可用性，我们就应该尽量减少 Elasticsearch 的不可用时间。 那么怎样提高 Elasticsearch 的高可用性呢？这时集群的作用就体现出来了。假如 Elasticsearch 只放在一台服务器上，即单机运行，假如这台主机突然断网了或者被攻击了，那么整个 Elasticsearch 的服务就不可用了。但如果改成 Elasticsearch 集群的话，有一台主机宕机了，还有其他的主机可以支撑，这样就仍然可以保证服务是可用的。 那可能有的小伙伴就会说了，那假如一台主机宕机了，那么不就无法访问这台主机的数据了吗？那假如我要访问的数据正好存在这台主机上，那不就获取不到了吗？难道其他的主机里面也存了一份一模一样的数据？那这岂不是很浪费吗？ 为了解答这个问题，这里就引出了 Elasticsearch 的信息存储机制了。首先解答上面的问题，一台主机宕机了，这台主机里面存的数据依然是可以被访问到的，因为在其他的主机上也有备份，但备份的时候也不是整台主机备份，是分片备份的，那这里就又引出了一个概念——分片。 分片，英文叫做 Shard，顾名思义，分片就是对数据切分成了多个部分。我们知道 Elasticsearch 中一个索引（Index）相当于是一个数据库，如存某网站的用户信息，我们就建一个名为 user 的索引。但索引存储的时候并不是整个存一起的，它是被分片存储的，Elasticsearch 默认会把一个索引分成五个分片，当然这个数字是可以自定义的。分片是数据的容器，数据保存在分片内，分片又被分配到集群内的各个节点里。当你的集群规模扩大或者缩小时， Elasticsearch 会自动的在各节点中迁移分片，使得数据仍然均匀分布在集群里，所以相当于一份数据被分成了多份并保存在不同的主机上。 那这还是没解决问题啊，如果一台主机挂掉了，那么这个分片里面的数据不就无法访问了？别的主机都是存储的其他的分片。其实是可以访问的，因为其他主机存储了这个分片的备份，叫做副本，这里就引出了另外一个概念——副本。 副本，英文叫做 Replica，同样顾名思义，副本就是对原分片的复制，和原分片的内容是一样的，Elasticsearch 默认会生成一份副本，所以相当于是五个原分片和五个分片副本，相当于一份数据存了两份，并分了十个分片，当然副本的数量也是可以自定义的。这时我们只需要将某个分片的副本存在另外一台主机上，这样当某台主机宕机了，我们依然还可以从另外一台主机的副本中找到对应的数据。所以从外部来看，数据结果是没有任何区别的。 一般来说，Elasticsearch 会尽量把一个索引的不同分片存储在不同的主机上，分片的副本也尽可能存在不同的主机上，这样可以提高容错率，从而提高高可用性。 但这时假如你只有一台主机，那不就没办法了吗？分片和副本其实是没意义的，一台主机挂掉了，就全挂掉了。</p>
                  <h3 id="健康状态"><a href="#健康状态" class="headerlink" title="健康状态"></a>健康状态</h3>
                  <p>针对一个索引，Elasticsearch 中其实有专门的衡量索引健康状况的标志，分为三个等级：</p>
                  <ul>
                    <li>green，绿色。这代表所有的主分片和副本分片都已分配。你的集群是 100% 可用的。</li>
                    <li>yellow，黄色。所有的主分片已经分片了，但至少还有一个副本是缺失的。不会有数据丢失，所以搜索结果依然是完整的。不过，你的高可用性在某种程度上被弱化。如果更多的分片消失，你就会丢数据了。所以可把 yellow 想象成一个需要及时调查的警告。</li>
                    <li>red，红色。至少一个主分片以及它的全部副本都在缺失中。这意味着你在缺少数据：搜索只能返回部分数据，而分配到这个分片上的写入请求会返回一个异常。</li>
                  </ul>
                  <p>如果你只有一台主机的话，其实索引的健康状况也是 yellow，因为一台主机，集群没有其他的主机可以防止副本，所以说，这就是一个不健康的状态，因此集群也是十分有必要的。</p>
                  <h3 id="存储空间"><a href="#存储空间" class="headerlink" title="存储空间"></a>存储空间</h3>
                  <p>另外，既然是群集，那么存储空间肯定也是联合起来的，假如一台主机的存储空间是固定的，那么集群它相对于单个主机也有更多的存储空间，可存储的数据量也更大。 所以综上所述，我们需要一个集群！</p>
                  <h2 id="详细了解-Elasticsearch-集群"><a href="#详细了解-Elasticsearch-集群" class="headerlink" title="详细了解 Elasticsearch 集群"></a>详细了解 Elasticsearch 集群</h2>
                  <p>接下来我们再来了解下集群的结构是怎样的。 首先我们应该清楚多台主机构成了一个集群，每台主机称作一个节点（Node）。 如图就是一个三节点的集群：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-04-23-52-42.png" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-04-23-52-42.png" alt=""></p>
                  <p>在图中，每个 Node 都有三个分片，其中 P 开头的代表 Primary 分片，即主分片，R 开头的代表 Replica 分片，即副本分片。所以图中主分片 1、2，副本分片 0 储存在 1 号节点，副本分片 0、1、2 储存在 2 号节点，主分片 0 和副本分片 1、2 储存在 3 号节点，一共是 3 个主分片和 6 个副本分片。同时我们还注意到 1 号节点还有个 MASTER 的标识，这代表它是一个主节点，它相比其他的节点更加特殊，它有权限控制整个集群，比如资源的分配、节点的修改等等。 这里就引出了一个概念就是节点的类型，我们可以将节点分为这么四个类型：</p>
                  <ul>
                    <li>主节点：即 Master 节点。主节点的主要职责是和集群操作相关的内容，如创建或删除索引，跟踪哪些节点是群集的一部分，并决定哪些分片分配给相关的节点。稳定的主节点对集群的健康是非常重要的。默认情况下任何一个集群中的节点都有可能被选为主节点。索引数据和搜索查询等操作会占用大量的cpu，内存，io资源，为了确保一个集群的稳定，分离主节点和数据节点是一个比较好的选择。虽然主节点也可以协调节点，路由搜索和从客户端新增数据到数据节点，但最好不要使用这些专用的主节点。一个重要的原则是，尽可能做尽量少的工作。</li>
                    <li>数据节点：即 Data 节点。数据节点主要是存储索引数据的节点，主要对文档进行增删改查操作，聚合操作等。数据节点对 CPU、内存、IO 要求较高，在优化的时候需要监控数据节点的状态，当资源不够的时候，需要在集群中添加新的节点。</li>
                    <li>负载均衡节点：也称作 Client 节点，也称作客户端节点。当一个节点既不配置为主节点，也不配置为数据节点时，该节点只能处理路由请求，处理搜索，分发索引操作等，从本质上来说该客户节点表现为智能负载平衡器。独立的客户端节点在一个比较大的集群中是非常有用的，他协调主节点和数据节点，客户端节点加入集群可以得到集群的状态，根据集群的状态可以直接路由请求。</li>
                    <li>预处理节点：也称作 Ingest 节点，在索引数据之前可以先对数据做预处理操作，所有节点其实默认都是支持 Ingest 操作的，也可以专门将某个节点配置为 Ingest 节点。</li>
                  </ul>
                  <p>以上就是节点几种类型，一个节点其实可以对应不同的类型，如一个节点可以同时成为主节点和数据节点和预处理节点，但如果一个节点既不是主节点也不是数据节点，那么它就是负载均衡节点。具体的类型可以通过具体的配置文件来设置。</p>
                  <h2 id="怎样搭建-Elasticsearch-集群"><a href="#怎样搭建-Elasticsearch-集群" class="headerlink" title="怎样搭建 Elasticsearch 集群"></a>怎样搭建 Elasticsearch 集群</h2>
                  <p>好，接下来我们就来动手搭建一个集群吧。 这里我一共拥有七台 Linux 主机，系统是 Ubuntu 16.04，都连接在一个内网中，IP 地址为：</p>
                  <figure class="highlight accesslog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">10.0.0.4</span></span><br><span class="line"><span class="number">10.0.0.5</span></span><br><span class="line"><span class="number">10.0.0.6</span></span><br><span class="line"><span class="number">10.0.0.7</span></span><br><span class="line"><span class="number">10.0.0.8</span></span><br><span class="line"><span class="number">10.0.0.9</span></span><br><span class="line"><span class="number">10.0.0.10</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>每台主机的存储空间是 1TB，内存是 13GB。 下面我们来一步步介绍如何用这几台主机搭建一个 Elasticsearch 集群，这里使用的 Elasticsearch 版本是 6.3.2，另外我们还需要安装 Kibana 用来可视化监控和管理 Elasticsearch 的相关配置和数据，使得集群的管理更加方便。 环境配置如下所示：</p>
                  <p>名称</p>
                  <p>内容</p>
                  <p>主机台数</p>
                  <p>7</p>
                  <p>主机内存</p>
                  <p>13GB</p>
                  <p>主机系统</p>
                  <p>Ubuntu 16.04</p>
                  <p>存储空间</p>
                  <p>1TB</p>
                  <p>Elasticsearch 版本</p>
                  <p>6.3.2</p>
                  <p>Java 版本</p>
                  <p>1.8</p>
                  <p>Kibana 版本</p>
                  <p>6.3.2</p>
                  <h3 id="安装-Java"><a href="#安装-Java" class="headerlink" title="安装 Java"></a>安装 Java</h3>
                  <p>Elasticsearch 是基于 Lucene 的，而 Lucene 又是基于 Java 的。所以第一步我们就需要在每台主机上安装 Java。 首先更新 Apt 源：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="keyword">get</span> <span class="keyword">update</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后安装 Java：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="keyword">get</span> install <span class="keyword">default</span>-jre</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好了之后可以检查下 Java 的版本：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">java -<span class="built_in">version</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的版本是 1.8，类似输出如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">openjdk version <span class="string">"1.8.0_171"</span></span><br><span class="line">OpenJDK Runtime Environment (build 1.8.0_171-8u171-b11-0ubuntu0.16.04.1-b11)</span><br><span class="line">OpenJDK 64-Bit<span class="built_in"> Server </span>VM (build 25.171-b11, mixed mode)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果看到上面的内容就说明安装成功了。 注意一定要每台主机都要安装。</p>
                  <h2 id="安装-Elasticsearch"><a href="#安装-Elasticsearch" class="headerlink" title="安装 Elasticsearch"></a>安装 Elasticsearch</h2>
                  <p>接下来我们来安装 Elasticsearch，同样是每台主机都需要安装。 首先需要添加 Apt-key：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">wget -qO - https://artifacts.elastic.co/GPG-<span class="meta">KEY</span>-elasticsearch | sudo apt-<span class="meta">key</span> <span class="meta">add</span> -</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后添加 Elasticsearch 的 Repository 定义：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">echo <span class="string">"deb https://artifacts.elastic.co/packages/6.x/apt stable main"</span> | sudo tee -a /etc/apt/sources.<span class="type">list</span>.d/elastic<span class="number">-6.</span>x.<span class="type">list</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来安装 Elasticsearch 即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> update </span><br><span class="line">sudo apt-<span class="builtin-name">get</span> install elasticsearch</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行完毕之后我们就完成了 Elasticsearch 的安装，注意还是要每台主机都要安装。</p>
                  <h3 id="配置-Elasticsearch"><a href="#配置-Elasticsearch" class="headerlink" title="配置 Elasticsearch"></a>配置 Elasticsearch</h3>
                  <p>这时我们只是每台主机都安装好了 Elasticsearch，接下来我们还需要将它们联系在一起构成一个集群。 安装完之后，Elasticsearch 的配置文件是 /etc/elasticsearch/elasticsearch.yml，接下来让我们编辑一下配置文件：</p>
                  <ul>
                    <li>集群的名称</li>
                  </ul>
                  <p>通过 cluster.name 可以配置集群的名称，集群是一个整体，因此名称都要一致，所有主机都配置成相同的名称，配置示例：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">cluster</span>.name: germey-es-clusters</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>节点的名称</li>
                  </ul>
                  <p>通过 node.name 可以配置每个节点的名称，每个节点都是集群的一部分，每个节点名称都不要相同，可以按照顺序编号，配置示例：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">node</span>.name:<span class="title"> es-node-1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其他的主机可以配置为 es-node-2、es-node-3 等。</p>
                  <ul>
                    <li>是否有资格成为主节点</li>
                  </ul>
                  <p>通过 node.master 可以配置该节点是否有资格成为主节点，如果配置为 true，则主机有资格成为主节点，配置为 false 则主机就不会成为主节点，可以去当数据节点或负载均衡节点。注意这里是有资格成为主节点，不是一定会成为主节点，主节点需要集群经过选举产生。这里我配置所有主机都可以成为主节点，因此都配置为 true，配置示例： node.master: true</p>
                  <ul>
                    <li>是否是数据节点</li>
                  </ul>
                  <p>通过 node.data 可以配置该节点是否为数据节点，如果配置为 true，则主机就会作为数据节点，注意主节点也可以作为数据节点，当 node.master 和 node.data 均为 false，则该主机会作为负载均衡节点。这里我配置所有主机都是数据节点，因此都配置为 true，配置示例： node.data: true</p>
                  <ul>
                    <li>数据和日志路径</li>
                  </ul>
                  <p>通过 path.data 和 path.logs 可以配置 Elasticsearch 的数据存储路径和日志存储路径，可以指定任意位置，这里我指定存储到 1T 硬盘对应的路径下，另外注意一下写入权限问题，配置示例： path.data: /datadrive/elasticsearch/data path.logs: /datadrive/elasticsearch/logs</p>
                  <ul>
                    <li>​设置访问的地址和端口</li>
                  </ul>
                  <p>我们需要设定 Elasticsearch 运行绑定的 Host，默认是无法公开访问的，如果设置为主机的公网 IP 或 0.0.0.0 就是可以公开访问的，这里我们可以都设置为公开访问或者部分主机公开访问，如果是公开访问就配置为：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">network</span><span class="selector-class">.host</span>: 0<span class="selector-class">.0</span><span class="selector-class">.0</span><span class="selector-class">.0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果不想被公开访问就不用配置。 另外还可以配置访问的端口，默认是 9200： http.port: 9200</p>
                  <ul>
                    <li>集群地址设置</li>
                  </ul>
                  <p>通过 discovery.zen.ping.unicast.hosts 可以配置集群的主机地址，配置之后集群的主机之间可以自动发现，这里我配置的是内网地址，配置示例：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">discovery</span><span class="selector-class">.zen</span><span class="selector-class">.ping</span><span class="selector-class">.unicast</span><span class="selector-class">.hosts</span>: <span class="selector-attr">[<span class="string">"10.0.0.4"</span>, <span class="string">"10.0.0.5"</span>, <span class="string">"10.0.0.6"</span>, <span class="string">"10.0.0.7"</span>, <span class="string">"10.0.0.8"</span>, <span class="string">"10.0.0.9"</span>, <span class="string">"10.0.0.10"</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里请改成你的主机对应的 IP 地址。</p>
                  <ul>
                    <li>节点数目相关配置</li>
                  </ul>
                  <p>为了防止集群发生“脑裂”，即一个集群分裂成多个，通常需要配置集群最少主节点数目，通常为 (可成为主节点的主机数目 / 2) + 1，例如我这边可以成为主节点的主机数目为 7，那么结果就是 4，配置示例：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">discovery</span><span class="selector-class">.zen</span><span class="selector-class">.minimum_master_nodes</span>: 4</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外还可以配置当最少几个节点回复之后，集群就正常工作，这里我设置为 4，可以酌情修改，配置示例：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">gateway.recover_after_nodes: <span class="number">4</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其他的暂时先不需要配置，保存即可。注意每台主机都需要配置。</p>
                  <h3 id="启动-Elasticsearch"><a href="#启动-Elasticsearch" class="headerlink" title="启动 Elasticsearch"></a>启动 Elasticsearch</h3>
                  <p>配置完成之后就可以在每台主机上分别启动 Elasticsearch 服务了，命令如下：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo<span class="keyword"> system</span>ctl daemon-reload</span><br><span class="line">sudo<span class="keyword"> system</span>ctl enable elasticsearch.service</span><br><span class="line">sudo<span class="keyword"> system</span>ctl start elasticsearch.service</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所有主机都启动之后，我们在任意主机上就可以查看到集群状态了，命令行如下：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl -XGET 'http://localhost:<span class="number">9200</span>/_cluster/<span class="keyword">state</span>?pretty'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>类似的输出如下：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="string">"cluster_name"</span>: <span class="string">"germey-es-clusters"</span>,</span><br><span class="line">    <span class="string">"compressed_size_in_bytes"</span>: <span class="number">20799</span>,</span><br><span class="line">    <span class="string">"version"</span>: <span class="number">658</span>,</span><br><span class="line">    <span class="string">"state_uuid"</span>: <span class="string">"a64wCwPnSueKRtVuKx8xRw"</span>,</span><br><span class="line">    <span class="string">"master_node"</span>: <span class="string">"73BQvOC2TpSXcr-IXBcDdg"</span>,</span><br><span class="line">    <span class="string">"blocks"</span>: &#123;&#125;,</span><br><span class="line">    <span class="string">"nodes"</span>: &#123;</span><br><span class="line">        <span class="string">"I2M80AP-T7yVP_AZPA0bpA"</span>: &#123;</span><br><span class="line">            <span class="string">"name"</span>: <span class="string">"es-node-1"</span>,</span><br><span class="line">            <span class="string">"ephemeral_id"</span>: <span class="string">"KpCG4jNvTUGKNHNwKKoMrA"</span>,</span><br><span class="line">            <span class="string">"transport_address"</span>: <span class="string">"10.0.0.4:9300"</span>,</span><br><span class="line">            <span class="string">"attributes"</span>: &#123;</span><br><span class="line">                <span class="string">"ml.machine_memory"</span>: <span class="string">"7308464128"</span>,</span><br><span class="line">                <span class="string">"ml.max_open_jobs"</span>: <span class="string">"20"</span>,</span><br><span class="line">                <span class="string">"xpack.installed"</span>: <span class="string">"true"</span>,</span><br><span class="line">                <span class="string">"ml.enabled"</span>: <span class="string">"true"</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="string">"73BQvOC2TpSXcr-IXBcDdg"</span>: &#123;</span><br><span class="line">            <span class="string">"name"</span>: <span class="string">"es-node-7"</span>,</span><br><span class="line">            <span class="string">"ephemeral_id"</span>: <span class="string">"Fs9v2XTASnGbqrM8g7IhAQ"</span>,</span><br><span class="line">            <span class="string">"transport_address"</span>: <span class="string">"10.0.0.10:9300"</span>,</span><br><span class="line">            <span class="string">"attributes"</span>: &#123;</span><br><span class="line">                <span class="string">"ml.machine_memory"</span>: <span class="string">"14695202816"</span>,</span><br><span class="line">                <span class="string">"ml.max_open_jobs"</span>: <span class="string">"20"</span>,</span><br><span class="line">                <span class="string">"xpack.installed"</span>: <span class="string">"true"</span>,</span><br><span class="line">                <span class="string">"ml.enabled"</span>: <span class="string">"true"</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">....</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这里输出了集群的相关信息，同时 nodes 字段里面包含了每个节点的详细信息，这样一个基本的集群就构建完成了。</p>
                  <h3 id="安装-Kibana"><a href="#安装-Kibana" class="headerlink" title="安装 Kibana"></a>安装 Kibana</h3>
                  <p>接下来我们需要安装一个 Kibana 来帮助可视化管理 Elasticsearch，依然还是通过 Apt 安装，只需要任意一台主机安装即可，因为集群是一体的，所以 Kibana 在任意一台主机只要能连接到 Elasticsearch 即可，安装命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> install kibana</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装之后修改 /etc/kibana/kibana.yml，设置公开访问和绑定的端口：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">server</span><span class="selector-class">.port</span>: 5601</span><br><span class="line"><span class="selector-tag">server</span><span class="selector-class">.host</span>: "0<span class="selector-class">.0</span><span class="selector-class">.0</span><span class="selector-class">.0</span>"</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后启动服务：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo<span class="keyword"> system</span>ctl daemon-reload</span><br><span class="line">sudo<span class="keyword"> system</span>ctl enable kibana.service</span><br><span class="line">sudo<span class="keyword"> system</span>ctl start kibana.service</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们可以在浏览器输入该台主机的 IP 加端口，查看 Kibana 管理页面了，类似如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-05-01-44-28.jpg" alt=""></p>
                  <p>这样 Kibana 可视化管理就配置成功了。</p>
                  <h3 id="配置认证"><a href="#配置认证" class="headerlink" title="配置认证"></a>配置认证</h3>
                  <p>现在集群已经初步搭建完成了，但是现在集群很危险，如果我们配置了可公网访问，那么它是可以被任何人操作的，比如储存数据，增删节点等，这是非常危险的，所以我们必须要设置访问权限。 在 Elasticsearch 中，配置认证是通过 X-Pack 插件实现的，幸运的是，我们不需要额外安装了，在 Elasticsearch 6.3.2 版本中，该插件是默认集成到 Elasticsearch 中的，所以我们只需要更改一部分设置就可以了。 首先我们需要升级 License，只有修改了高级版 License 才能使用X-Pack 的权限认证功能。 在 Kibana 中访问 Management -&gt; Elasticsearch -&gt; License Management，点击右侧的升级 License 按钮，可以免费试用 30 天的高级 License，升级完成之后页面会显示如下：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-03-16-45-10.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-03-16-45-10.jpg" alt=""></p>
                  <p>另外还可以使用 API 来更新 License，详情可以参考官方文档：<a href="https://www.elastic.co/guide/en/elasticsearch/reference/6.2/update-license.html" target="_blank" rel="noopener">https://www.elastic.co/guide/en/elasticsearch/reference/6.2/update-license.html</a>。 然后每台主机需要修改 /etc/elasticsearch/elasticsearch.yml 文件，开启 Auth 认证功能：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">xpack</span><span class="selector-class">.security</span><span class="selector-class">.enabled</span>: <span class="selector-tag">true</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后设置 elastic、kibana、logstash_system 三个用户的密码，任意一台主机修改之后，一台修改，多台生效，命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="regexp">/usr/</span>share<span class="regexp">/elasticsearch/</span>bin<span class="regexp">/elasticsearch-setup-passwords interactive</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后会依次提示设置这三个用户的密码并确认，一共需要输入六次密码，完成之后就成功设置好了密码了。 修改完成之后重启 Elasticsearch 和 Kibana 服务：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">systemctl</span> <span class="selector-tag">restart</span> <span class="selector-tag">elasticsearch</span><span class="selector-class">.service</span></span><br><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">systemctl</span> <span class="selector-tag">restart</span> <span class="selector-tag">kibana</span><span class="selector-class">.service</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时再访问 Kibana 就会跳转到登录页面了：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-03-17-18-28.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-03-17-18-28.jpg" alt=""></p>
                  <p>可以使用 elastic 用户登录，它的角色是超级管理员，登录之后就可以重新进入 Kibana 的管理页面。 我们还可以自行修改和添加账户，在 Management -&gt; Security -&gt; User/Roles 里面：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-03-17-20-05.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-03-17-20-05.jpg" alt=""></p>
                  <p>例如这里添加一个超级管理员的账户：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-03-17-21-44.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-03-17-21-44.jpg" alt=""></p>
                  <p>这样以后我们就可以使用新添加的用户来登录和访问了。 另外修改权限认证之后，Elasticsearch 也不能直接访问了，我们也必须输入用户密码才可以访问和调用其 API，保证了安全性。</p>
                  <h3 id="开启内存锁定"><a href="#开启内存锁定" class="headerlink" title="开启内存锁定"></a>开启内存锁定</h3>
                  <p>系统默认会进行内存交换，这样会导致 Elasticsearch 的性能变差，我们查看下内存锁定状态，在任意一台主机上的访问 <a href="http://ip:port/_nodes?filter_path=**.mlockall：">http://ip:port/_nodes?filter_path=**.mlockall：</a> 可以看到如下结果：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"nodes"</span>: &#123;</span><br><span class="line">        <span class="attr">"73BQvOC2TpSXcr-IXBcDdg"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"9tRr4nFDT_2rErLLQB2dIQ"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"hskSDv_JQlCUnjp_INI8Kg"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"LgaRuqXBTZaBdDGAktFWJA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"ZcsZgowERzuvpqVbYOgOEA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"I2M80AP-T7yVP_AZPA0bpA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">false</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"_mSmfhUtQiqhzTKZ7u75Dw"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这代表内存交换没有开启，会影响 Elasticsearch 的性能，所以我们需要开启内存物理地址锁定，每台主机需要修改 /etc/elasticsearch/elasticsearch.yml 文件，修改如下配置：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">bootstrap.<span class="string">memory_lock:</span> <span class="literal">true</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但这样修改之后重新启动是会报错的，Elasticsearch 无法正常启动，查看日志，报错如下：</p>
                  <figure class="highlight inform7">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">[1]</span> bootstrap checks failed</span><br><span class="line"><span class="comment">[1]</span>: memory locking requested for elasticsearch process but memory <span class="keyword">is</span> not <span class="keyword">locked</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要修改两个地方，第一个是 /etc/security/limits.conf，添加如下内容：</p>
                  <figure class="highlight asciidoc">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="bullet">* </span>soft nofile 65536</span><br><span class="line"><span class="bullet">* </span>hard nofile 65536</span><br><span class="line"><span class="bullet">* </span>soft nproc 32000</span><br><span class="line"><span class="bullet">* </span>hard nproc 32000</span><br><span class="line"><span class="bullet">* </span>hard memlock unlimited</span><br><span class="line"><span class="bullet">* </span>soft memlock unlimited</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外还需要修改 /etc/systemd/system.conf，修改如下内容：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">DefaultLimitNOFILE</span>=<span class="number">65536</span></span><br><span class="line"><span class="attr">DefaultLimitNPROC</span>=<span class="number">32000</span></span><br><span class="line"><span class="attr">DefaultLimitMEMLOCK</span>=infinity</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>详细的解释可以参考：<a href="https://segmentfault.com/a/1190000014891856" target="_blank" rel="noopener">https://segmentfault.com/a/1190000014891856</a>。 修改之后重启 Elasticsearch 服务：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">systemctl</span> <span class="selector-tag">restart</span> <span class="selector-tag">elasticsearch</span><span class="selector-class">.service</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>重新访问刚才的地址，即可发现每台主机的物理地址锁定都被打开了：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"nodes"</span>: &#123;</span><br><span class="line">        <span class="attr">"73BQvOC2TpSXcr-IXBcDdg"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"9tRr4nFDT_2rErLLQB2dIQ"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"hskSDv_JQlCUnjp_INI8Kg"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"LgaRuqXBTZaBdDGAktFWJA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"ZcsZgowERzuvpqVbYOgOEA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"I2M80AP-T7yVP_AZPA0bpA"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"_mSmfhUtQiqhzTKZ7u75Dw"</span>: &#123;</span><br><span class="line">            <span class="attr">"process"</span>: &#123;</span><br><span class="line">                <span class="attr">"mlockall"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就又解决了性能的问题。</p>
                  <h3 id="安装分词插件"><a href="#安装分词插件" class="headerlink" title="安装分词插件"></a>安装分词插件</h3>
                  <p>另外还推荐安装中文分词插件，这样可以对中文进行全文索引，安装命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo <span class="regexp">/usr/</span>share<span class="regexp">/elasticsearch/</span>bin<span class="regexp">/elasticsearch-plugin install https:/</span><span class="regexp">/github.com/m</span>edcl<span class="regexp">/elasticsearch-analysis-ik/</span>releases<span class="regexp">/download/</span>v6.<span class="number">3.2</span><span class="regexp">/elasticsearch-analysis-ik-6.3.2.zip</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完之后需要重启 Elasticsearch 服务：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">sudo</span> <span class="selector-tag">systemctl</span> <span class="selector-tag">restart</span> <span class="selector-tag">elasticsearch</span><span class="selector-class">.service</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="主机监控"><a href="#主机监控" class="headerlink" title="主机监控"></a>主机监控</h3>
                  <p>到此为止，我们的 Elasticsearch 集群就搭建完成了。 最后我们看下 Kibana 的部分功能，看下整个 Elasticsearch 有没有在正常工作。 访问 Kibana，打开 Management -&gt; Elasticsearch -&gt;Index Management，即可看到当前有的一些索引和状态：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-05-02-29-07.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-05-02-29-07.jpg" alt=""></p>
                  <p>打开 Monitoring，可以查看 Elasticsearch 和 Kibana 的状态：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-05-02-27-33.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-05-02-27-33.jpg" alt=""></p>
                  <p>进一步点击 Nodes，可以查看各个节点的状态：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-05-02-28-01.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-05-02-28-01.jpg" alt=""></p>
                  <p>打开任意节点，可以查看当前资源状况变化：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/2018-08-05-02-28-30.jpg" alt=""><img src="https://mp.weixin.qq.com/cgi-bin/assets/2018-08-05-02-28-30.jpg" alt=""></p>
                  <p>另外还有一些其他的功能如可视化、图表、搜索等等，这里就不再一一列举了，更多功能可以详细了解 Kibana。 以上都是自己在安装过程中趟过的坑，如有疏漏，还望指正。 还有更多的 Elasticsearch 相关的内容可以参考官方文档：<a href="https://www.elastic.co/guide/index.html" target="_blank" rel="noopener">https://www.elastic.co/guide/index.html</a>。</p>
                  <h2 id="参考资料"><a href="#参考资料" class="headerlink" title="参考资料"></a>参考资料</h2>
                  <ul>
                    <li><a href="https://www.elastic.co/guide/en/x-pack/current/security-getting-started.html" target="_blank" rel="noopener">https://www.elastic.co/guide/en/x-pack/current/security-getting-started.html</a></li>
                    <li><a href="https://segmentfault.com/a/1190000014891856" target="_blank" rel="noopener">https://segmentfault.com/a/1190000014891856</a></li>
                    <li><a href="https://blog.csdn.net/a19860903/article/details/72467996" target="_blank" rel="noopener">https://blog.csdn.net/a19860903/article/details/72467996</a></li>
                    <li><a href="https://logz.io/blog/elasticsearch-cluster-tutorial/" target="_blank" rel="noopener">https://logz.io/blog/elasticsearch-cluster-tutorial/</a></li>
                    <li><a href="https://es.xiaoleilu.com/020_Distributed_Cluster/30_Scale_more.html" target="_blank" rel="noopener">https://es.xiaoleilu.com/020_Distributed_Cluster/30_Scale_more.html</a></li>
                    <li><a href="https://blog.csdn.net/archer119/article/details/76589189" target="_blank" rel="noopener">https://blog.csdn.net/archer119/article/details/76589189</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-08-05 02:57:01" itemprop="dateCreated datePublished" datetime="2018-08-05T02:57:01+08:00">2018-08-05</time>
                </span>
                <span id="/6255.html" class="post-meta-item leancloud_visitors" data-flag-title="Ubuntu 搭建 Elasticsearch 6 集群流程" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>11k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>10 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6217.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6217.html" class="post-title-link" itemprop="url">快来学习怎么可视化监控你的爬虫</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="大家好，我是四毛，下面是我的个人公众号，欢迎关注。有问题的可以私信我，看到就会回复。"><a href="#大家好，我是四毛，下面是我的个人公众号，欢迎关注。有问题的可以私信我，看到就会回复。" class="headerlink" title="大家好，我是四毛，下面是我的个人公众号，欢迎关注。有问题的可以私信我，看到就会回复。"></a>大家好，我是四毛，下面是我的个人公众号，欢迎关注。有问题的可以私信我，看到就会回复。</h1>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/201802111155445124jhQyer.yasuotu.gif" alt=""> </p>
                  <h2 id="更新-2018年08月03日14-39-32"><a href="#更新-2018年08月03日14-39-32" class="headerlink" title="更新 2018年08月03日14:39:32"></a>更新 2018年08月03日14:39:32</h2>
                  <blockquote>
                    <p>其实可以利用scrapy的扩展展示更多的数据，立个flag，后面更新上来</p>
                  </blockquote>
                  <p> 好，开始今天的文章。 今天主要是来说一下怎么可视化来<strong>监控你的爬虫的状态</strong>。 相信大家在跑爬虫的过程中，也会好奇自己养的爬虫一分钟可以<strong>爬多少页面</strong>，<strong>多大的数据量</strong>，当然查询的方式多种多样。今天我来讲一种可视化的方法。</p>
                  <h3 id="关于爬虫数据在mongodb里的版本我写了一个可以热更新配置的版本，即添加了新的爬虫配置以后，不用重启程序，即可获取刚刚添加的爬虫的状态数据，大家可以通过关注我的公众号以后，-回复“可视化”即可获取脚本地址。"><a href="#关于爬虫数据在mongodb里的版本我写了一个可以热更新配置的版本，即添加了新的爬虫配置以后，不用重启程序，即可获取刚刚添加的爬虫的状态数据，大家可以通过关注我的公众号以后，-回复“可视化”即可获取脚本地址。" class="headerlink" title="关于爬虫数据在mongodb里的版本我写了一个可以热更新配置的版本，即添加了新的爬虫配置以后，不用重启程序，即可获取刚刚添加的爬虫的状态数据，大家可以通过关注我的公众号以后， 回复“可视化”即可获取脚本地址。"></a><strong>关于爬虫数据在mongodb里的版本我写了一个可以热更新配置的版本，即添加了新的爬虫配置以后，不用重启程序，即可获取刚刚添加的爬虫的状态数据，大家可以通过关注我的公众号以后， 回复“可视化”即可获取脚本地址</strong>。</h3>
                  <h2 id="1-成品图"><a href="#1-成品图" class="headerlink" title="1.成品图"></a>1.成品图</h2>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_164.png" alt=""> 这个是监控服务器网速的最后成果，显示的是下载与上传的网速，单位为M。爬虫的原理都是一样的，只不过将数据存到InfluxDB的方式不一样而已， 如下图。</p>
                  <h3 id=""><a href="#" class="headerlink" title=""></a><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_172.png" alt=""></h3>
                  <p>可以实现对爬虫数量，增量，大小，大小增量的实时监控。 </p>
                  <h2 id="2-环境"><a href="#2-环境" class="headerlink" title="2. 环境"></a>2. 环境</h2>
                  <ul>
                    <li><strong>InfluxDb</strong>，是目前比较流行的时间序列数据库；</li>
                    <li><strong>Grafana</strong>，一个可视化面板（Dashboard），有着非常漂亮的图表和布局展示，功能齐全的度量仪表盘和图形编辑器，支持Graphite、zabbix、InfluxDB、Prometheus和OpenTSDB作为数据源</li>
                    <li><strong>Ubuntu</strong></li>
                    <li>
                      <p><strong>influxdb</strong>（pip install influxdb)</p>
                    </li>
                    <li>
                      <p><strong>Python 2.7</strong></p>
                    </li>
                  </ul>
                  <h2 id="3-原理"><a href="#3-原理" class="headerlink" title="3. 原理"></a>3. 原理</h2>
                  <blockquote>
                    <p>获取要展示的数据，包含当前的时间数据，存到InfluxDb里面，然后再到Grafana里面进行相应的配置即可展示；</p>
                  </blockquote>
                  <h2 id="4-安装"><a href="#4-安装" class="headerlink" title="4. 安装"></a>4. 安装</h2>
                  <h3 id="4-1-Grafana安装"><a href="#4-1-Grafana安装" class="headerlink" title="4.1 Grafana安装"></a>4.1 Grafana安装</h3>
                  <p> <a href="http://docs.grafana.org/installation/debian/" target="_blank" rel="noopener">官方安装指导</a> 安装好以后，打开本地的3000端口，即可进入管理界面，用户名与密码都是<strong>admin</strong>。</p>
                  <h3 id="4-2-InfulxDb安装"><a href="#4-2-InfulxDb安装" class="headerlink" title="4.2 InfulxDb安装"></a>4.2 InfulxDb安装</h3>
                  <p>这个安装就网上自己找吧，有很多的配置我都没有配置，就不在这里误人子弟了。</p>
                  <h2 id="5-InfluxDb简单操作"><a href="#5-InfluxDb简单操作" class="headerlink" title="5. InfluxDb简单操作"></a>5. InfluxDb简单操作</h2>
                  <p>碰到了数据库，肯定要把增删改查学会了啊， 和sql几乎一样，只有一丝丝的区别，具体操作，大家可以参考官方的文档。</p>
                  <ul>
                    <li><strong>influx</strong> 进入命令行</li>
                    <li><strong>CREATE DATABASE test</strong> 创建数据库</li>
                    <li><strong>show databases</strong> 查看数据库</li>
                    <li><strong>use test</strong> 使用数据库</li>
                    <li><strong>show series</strong> 看表</li>
                    <li><strong>select * from table_test</strong> 选择数据</li>
                    <li><strong>DROP MEASUREMENT table_test</strong> 删表</li>
                  </ul>
                  <h2 id="6-存数据"><a href="#6-存数据" class="headerlink" title="6. 存数据"></a>6. 存数据</h2>
                  <p>InfluxDb数据库的数据有一定的格式，因为我都是利用python库进行相关操作，所以下面将在python中的格式展示一下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">json_body</span> = [</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="string">"measurement"</span>: <span class="string">"crawler"</span>,</span><br><span class="line">        <span class="string">"time"</span>: current_time,</span><br><span class="line">        <span class="string">"tags"</span>: &#123;</span><br><span class="line">            <span class="string">"spider_name"</span>: collection_name</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="string">"fields"</span>: &#123;</span><br><span class="line">            <span class="string">"count"</span>: current_count,</span><br><span class="line">            <span class="string">"increase_count"</span>: increase_amount,</span><br><span class="line">            <span class="string">"size"</span>: co_size,</span><br><span class="line">            <span class="string">"increase_size"</span>: increase_co_size</span><br><span class="line"></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中:</p>
                  <ul>
                    <li>measurement, 表名</li>
                    <li>time，时间</li>
                    <li>tags，标签</li>
                    <li>fields，字段</li>
                  </ul>
                  <p>可以看到，就是个列表里面，嵌套了一个字典。其中，对于时间字段，有特殊要求，可以参考<a href="https://stackoverflow.com/questions/32090883/how-to-use-time-field-in-adding-metrics-data-to-the-influx-db" target="_blank" rel="noopener">这里</a>， 下面是python实现方法：</p>
                  <figure class="highlight cos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from datetime import datetime</span><br><span class="line">current_time = datetime.utcnow().strftime('<span class="built_in">%Y</span>-<span class="built_in">%m</span>-<span class="built_in">%dT</span><span class="built_in">%H</span>:<span class="built_in">%M</span>:<span class="built_in">%SZ</span>')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以，到这里，如何将爬虫的相关属性存进去呢？以MongoDB为例</p>
                  <figure class="highlight nix">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">mongodb_client</span> = pymongo.MongoClient(uri)</span><br><span class="line">    for db_name, collection_name <span class="keyword">in</span> dbs_and_cos.iteritems():</span><br><span class="line">        <span class="comment"># 数据库操作</span></span><br><span class="line">        <span class="attr">db</span> = mongodb_client[db_name]</span><br><span class="line">        <span class="attr">co</span> = db[collection_name]</span><br><span class="line">        <span class="comment"># 集合大小</span></span><br><span class="line">        <span class="attr">co_size</span> = round(float(db.command(<span class="string">"collstats"</span>, collection_name).get('size')) / <span class="number">1024</span> / <span class="number">1024</span>, <span class="number">2</span>)</span><br><span class="line">        <span class="comment"># 集合内数据条数</span></span><br><span class="line">        <span class="attr">current_count</span> = co.count()</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 初始化，当程序刚执行时，初始量就设置为第一次执行时获取的数据</span></span><br><span class="line">        <span class="attr">init_count</span> = _count_dict.get(collection_name, current_count)</span><br><span class="line">        <span class="comment"># 初始化，当程序刚执行时，初始量就设置为第一次执行时获取的数据大小</span></span><br><span class="line">        <span class="attr">init_size</span> = _size_dict.get(collection_name, co_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 条数增长量</span></span><br><span class="line">        <span class="attr">increase_amount</span> = current_count - init_count</span><br><span class="line">        <span class="comment"># 集合大小增长量</span></span><br><span class="line">        <span class="attr">increase_co_size</span> = co_size - init_size</span><br><span class="line"></span><br><span class="line">        <span class="attr">current_time</span> = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 赋值</span></span><br><span class="line">        _size_dict[collection_name] = co_size</span><br><span class="line">        _count_dict[collection_name] = current_count</span><br><span class="line"></span><br><span class="line">        <span class="attr">json_body</span> = [</span><br><span class="line">            &#123;</span><br><span class="line">                <span class="string">"measurement"</span>: <span class="string">"crawler"</span>,</span><br><span class="line">                <span class="string">"time"</span>: current_time,</span><br><span class="line">                <span class="string">"tags"</span>: &#123;</span><br><span class="line">                    <span class="string">"spider_name"</span>: collection_name</span><br><span class="line">                &#125;,</span><br><span class="line">                <span class="string">"fields"</span>: &#123;</span><br><span class="line">                    <span class="string">"count"</span>: current_count,</span><br><span class="line">                    <span class="string">"increase_count"</span>: increase_amount,</span><br><span class="line">                    <span class="string">"size"</span>: co_size,</span><br><span class="line">                    <span class="string">"increase_size"</span>: increase_co_size</span><br><span class="line"></span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        ]</span><br><span class="line">        print json_body</span><br><span class="line">        client.write_points(json_body)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>完整代码，关注上面的公众号，发送“”可视化“”即可获取。 那么现在我们已经往数据里存了数据了，那么接下来要做的就是把存的数据展示出来。</p>
                  <h2 id="7-展示数据"><a href="#7-展示数据" class="headerlink" title="7.展示数据"></a>7.展示数据</h2>
                  <h3 id="7-1-配置数据源"><a href="#7-1-配置数据源" class="headerlink" title="7.1 配置数据源"></a>7.1 配置数据源</h3>
                  <p>以admin登录到Grafana的后台后，我们首先需要配置一下数据源。点击左边栏的最下面的按钮，然后点击DATA SOURCES，这样就可以进入下面的页面： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_167.png" alt=""> 点击ADD DATA SOURCE，进行配置即可，如下图： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_168.png" alt=""> 其中，name自行设定；Type 选择InfluxDB；url为默认的<a href="http://localhost:8086，">http://localhost:8086，</a> 其他的因为我前面没有进行配置，所以默认的即可。然后在InfluxDB Details里的填入Database名，最后点击测试，如果没有报错的话，则可以进入下一步的展示数据了；</p>
                  <h3 id="7-2-展示数据"><a href="#7-2-展示数据" class="headerlink" title="7.2 展示数据"></a>7.2 展示数据</h3>
                  <p>点击左边栏的+号，然后点击GRAPH <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_166.png" alt=""> 接着点击下图中的edit进入编辑页面： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_169.png" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/08/Selection_170.png" alt=""> 从上图中可以发现：</p>
                  <ul>
                    <li>中间板块是最后的数据展示</li>
                    <li>下面是数据的设置项</li>
                    <li>右上角是展示时间的设置板块，在这里可以选择要展示多久的数据</li>
                  </ul>
                  <h4 id="7-2-1-配置数据"><a href="#7-2-1-配置数据" class="headerlink" title="7.2.1 配置数据"></a>7.2.1 配置数据</h4>
                  <ol>
                    <li>在Data Source中选择刚刚在配置数据源的时候配置的NAME字段，而不是database名。</li>
                    <li>接着在下面选择要展示的数据。看着就很熟悉是不是，完全是sql语句的可视化。同时，当我们的数据放到相关的字段上的时候，双击，就会把可以选择的项展示出来了，我们要做的就是直接选择即可；</li>
                    <li>设置右上角的时间，则可以让数据实时进行更新与展示</li>
                  </ol>
                  <p>因为下面的配置实质就是sql查询语句，所以大家按照自己的需求，进行选择配置即可，当配置完以后，就可以在中间的面板里面看到数据了。 </p>
                  <h2 id="8-总结"><a href="#8-总结" class="headerlink" title="8. 总结"></a>8. 总结</h2>
                  <p>到这里，本篇文章就结束了。其中，对于Grafana的操作我没有介绍的很详细，因为本篇主要讲的是怎么利用这几个工具完成我们的任务。 同时，里面的功能确实很多，还有可以安装的插件。我自己目前还是仅仅对于用到的部分比较了解，所以大家可以查询官方的或者别的教程资料来对Grafana进行更深入的了解，制作出更加好看的可视化作品来。 最后，关注公众号，回复“可视化” 即可获取本文代码哦</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-08-02 19:24:36" itemprop="dateCreated datePublished" datetime="2018-08-02T19:24:36+08:00">2018-08-02</time>
                </span>
                <span id="/6217.html" class="post-meta-item leancloud_visitors" data-flag-title="快来学习怎么可视化监控你的爬虫" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6214.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6214.html" class="post-title-link" itemprop="url">Elasticsearch 基本介绍及其与 Python 的对接实现</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="什么是-Elasticsearch"><a href="#什么是-Elasticsearch" class="headerlink" title="什么是 Elasticsearch"></a>什么是 Elasticsearch</h2>
                  <p>想查数据就免不了搜索，搜索就离不开搜索引擎，百度、谷歌都是一个非常庞大复杂的搜索引擎，他们几乎索引了互联网上开放的所有网页和数据。然而对于我们自己的业务数据来说，肯定就没必要用这么复杂的技术了，如果我们想实现自己的搜索引擎，方便存储和检索，Elasticsearch 就是不二选择，它是一个全文搜索引擎，可以快速地储存、搜索和分析海量数据。</p>
                  <h2 id="为什么要用-Elasticsearch"><a href="#为什么要用-Elasticsearch" class="headerlink" title="为什么要用 Elasticsearch"></a>为什么要用 Elasticsearch</h2>
                  <p>Elasticsearch 是一个开源的搜索引擎，建立在一个全文搜索引擎库 Apache Lucene™ 基础之上。 那 Lucene 又是什么？Lucene 可能是目前存在的，不论开源还是私有的，拥有最先进，高性能和全功能搜索引擎功能的库，但也仅仅只是一个库。要用上 Lucene，我们需要编写 Java 并引用 Lucene 包才可以，而且我们需要对信息检索有一定程度的理解才能明白 Lucene 是怎么工作的，反正用起来没那么简单。 那么为了解决这个问题，Elasticsearch 就诞生了。Elasticsearch 也是使用 Java 编写的，它的内部使用 Lucene 做索引与搜索，但是它的目标是使全文检索变得简单，相当于 Lucene 的一层封装，它提供了一套简单一致的 RESTful API 来帮助我们实现存储和检索。 所以 Elasticsearch 仅仅就是一个简易版的 Lucene 封装吗？那就大错特错了，Elasticsearch 不仅仅是 Lucene，并且也不仅仅只是一个全文搜索引擎。 它可以被下面这样准确的形容：</p>
                  <ul>
                    <li>一个分布式的实时文档存储，每个字段可以被索引与搜索</li>
                    <li>一个分布式实时分析搜索引擎</li>
                    <li>能胜任上百个服务节点的扩展，并支持 PB 级别的结构化或者非结构化数据</li>
                  </ul>
                  <p>总之，是一个相当牛逼的搜索引擎，维基百科、Stack Overflow、GitHub 都纷纷采用它来做搜索。</p>
                  <h2 id="Elasticsearch-的安装"><a href="#Elasticsearch-的安装" class="headerlink" title="Elasticsearch 的安装"></a>Elasticsearch 的安装</h2>
                  <p>我们可以到 Elasticsearch 的官方网站下载 Elasticsearch：<a href="https://www.elastic.co/downloads/elasticsearch" target="_blank" rel="noopener">https://www.elastic.co/downloads/elasticsearch</a>，同时官网也附有安装说明。 首先把安装包下载下来并解压，然后运行 bin/elasticsearch（Mac 或 Linux）或者 bin\elasticsearch.bat (Windows) 即可启动 Elasticsearch 了。 我使用的是 Mac，Mac 下个人推荐使用 Homebrew 安装：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">brew </span><span class="keyword">install </span>elasticsearch</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>Elasticsearch 默认会在 9200 端口上运行，我们打开浏览器访问 <a href="http://localhost:9200/" target="_blank" rel="noopener">http://localhost:9200/</a> 就可以看到类似内容：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"name"</span> : <span class="string">"atntrTf"</span>,</span><br><span class="line">  <span class="attr">"cluster_name"</span> : <span class="string">"elasticsearch"</span>,</span><br><span class="line">  <span class="attr">"cluster_uuid"</span> : <span class="string">"e64hkjGtTp6_G2h1Xxdv5g"</span>,</span><br><span class="line">  <span class="attr">"version"</span> : &#123;</span><br><span class="line">    <span class="attr">"number"</span>: <span class="string">"6.2.4"</span>,</span><br><span class="line">    <span class="attr">"build_hash"</span>: <span class="string">"ccec39f"</span>,</span><br><span class="line">    <span class="attr">"build_date"</span>: <span class="string">"2018-04-12T20:37:28.497551Z"</span>,</span><br><span class="line">    <span class="attr">"build_snapshot"</span>: <span class="literal">false</span>,</span><br><span class="line">    <span class="attr">"lucene_version"</span>: <span class="string">"7.2.1"</span>,</span><br><span class="line">    <span class="attr">"minimum_wire_compatibility_version"</span>: <span class="string">"5.6.0"</span>,</span><br><span class="line">    <span class="attr">"minimum_index_compatibility_version"</span>: <span class="string">"5.0.0"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"tagline"</span> : <span class="string">"You Know, for Search"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果看到这个内容，就说明 Elasticsearch 安装并启动成功了，这里显示我的 Elasticsearch 版本是 6.2.4 版本，版本很重要，以后安装一些插件都要做到版本对应才可以。 接下来我们来了解一下 Elasticsearch 的基本概念以及和 Python 的对接。</p>
                  <h2 id="Elasticsearch-相关概念"><a href="#Elasticsearch-相关概念" class="headerlink" title="Elasticsearch 相关概念"></a>Elasticsearch 相关概念</h2>
                  <p>在 Elasticsearch 中有几个基本的概念，如节点、索引、文档等等，下面来分别说明一下，理解了这些概念对熟悉 Elasticsearch 是非常有帮助的。</p>
                  <h3 id="Node-和-Cluster"><a href="#Node-和-Cluster" class="headerlink" title="Node 和 Cluster"></a>Node 和 Cluster</h3>
                  <p>Elasticsearch 本质上是一个分布式数据库，允许多台服务器协同工作，每台服务器可以运行多个 Elasticsearch 实例。 单个 Elasticsearch 实例称为一个节点（Node）。一组节点构成一个集群（Cluster）。</p>
                  <h3 id="Index"><a href="#Index" class="headerlink" title="Index"></a>Index</h3>
                  <p>Elasticsearch 会索引所有字段，经过处理后写入一个反向索引（Inverted Index）。查找数据的时候，直接查找该索引。 所以，Elasticsearch 数据管理的顶层单位就叫做 Index（索引），其实就相当于 MySQL、MongoDB 等里面的数据库的概念。另外值得注意的是，每个 Index （即数据库）的名字必须是小写。</p>
                  <h3 id="Document"><a href="#Document" class="headerlink" title="Document"></a>Document</h3>
                  <p>Index 里面单条的记录称为 Document（文档）。许多条 Document 构成了一个 Index。 Document 使用 JSON 格式表示，下面是一个例子。 同一个 Index 里面的 Document，不要求有相同的结构（scheme），但是最好保持相同，这样有利于提高搜索效率。</p>
                  <h3 id="Type"><a href="#Type" class="headerlink" title="Type"></a>Type</h3>
                  <p>Document 可以分组，比如 weather 这个 Index 里面，可以按城市分组（北京和上海），也可以按气候分组（晴天和雨天）。这种分组就叫做 Type，它是虚拟的逻辑分组，用来过滤 Document，类似 MySQL 中的数据表，MongoDB 中的 Collection。 不同的 Type 应该有相似的结构（Schema），举例来说，id 字段不能在这个组是字符串，在另一个组是数值。这是与关系型数据库的表的一个区别。性质完全不同的数据（比如 products 和 logs）应该存成两个 Index，而不是一个 Index 里面的两个 Type（虽然可以做到）。 根据规划，Elastic 6.x 版只允许每个 Index 包含一个 Type，7.x 版将会彻底移除 Type。</p>
                  <h3 id="Fields"><a href="#Fields" class="headerlink" title="Fields"></a>Fields</h3>
                  <p>即字段，每个 Document 都类似一个 JSON 结构，它包含了许多字段，每个字段都有其对应的值，多个字段组成了一个 Document，其实就可以类比 MySQL 数据表中的字段。 在 Elasticsearch 中，文档归属于一种类型（Type），而这些类型存在于索引（Index）中，我们可以画一些简单的对比图来类比传统关系型数据库：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">R<span class="function"><span class="title">elational</span> DB -&gt;</span> D<span class="function"><span class="title">atabases</span> -&gt;</span> T<span class="function"><span class="title">ables</span> -&gt;</span> R<span class="function"><span class="title">ows</span> -&gt;</span> Columns</span><br><span class="line">E<span class="function"><span class="title">lasticsearch</span> -&gt;</span> I<span class="function"><span class="title">ndices</span>   -&gt;</span> T<span class="function"><span class="title">ypes</span>  -&gt;</span> D<span class="function"><span class="title">ocuments</span> -&gt;</span> Fields</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上就是 Elasticsearch 里面的一些基本概念，通过和关系性数据库的对比更加有助于理解。</p>
                  <h2 id="Python-对接-Elasticsearch"><a href="#Python-对接-Elasticsearch" class="headerlink" title="Python 对接 Elasticsearch"></a>Python 对接 Elasticsearch</h2>
                  <p>Elasticsearch 实际上提供了一系列 Restful API 来进行存取和查询操作，我们可以使用 curl 等命令来进行操作，但毕竟命令行模式没那么方便，所以这里我们就直接介绍利用 Python 来对接 Elasticsearch 的相关方法。 Python 中对接 Elasticsearch 使用的就是一个同名的库，安装方式非常简单：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> elasticsearch</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>官方文档是：<a href="https://elasticsearch-py.readthedocs.io/" target="_blank" rel="noopener">https://elasticsearch-py.readthedocs.io/</a>，所有的用法都可以在里面查到，文章后面的内容也是基于官方文档来的。</p>
                  <h3 id="创建-Index"><a href="#创建-Index" class="headerlink" title="创建 Index"></a>创建 Index</h3>
                  <p>我们先来看下怎样创建一个索引（Index），这里我们创建一个名为 news 的索引：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">result = es.indices.create(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">ignore</span>=400)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果创建成功，会返回如下结果：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="symbol">'acknowledged</span>': <span class="literal">True</span>, <span class="symbol">'shards_acknowledged</span>': <span class="literal">True</span>, <span class="symbol">'index</span>': <span class="symbol">'news</span>'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果是 JSON 格式，其中的 acknowledged 字段表示创建操作执行成功。 但这时如果我们再把代码执行一次的话，就会返回如下结果：</p>
                  <figure class="highlight sml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="symbol">'error'</span>: &#123;<span class="symbol">'root_cause'</span>: [&#123;<span class="symbol">'type'</span>: <span class="symbol">'resource_already_exists_exception'</span>, <span class="symbol">'reason'</span>: <span class="symbol">'index</span> [news/<span class="type">QM6yz2W8QE</span>-bflKhc5oThw] already exists', <span class="symbol">'index_uuid'</span>: <span class="symbol">'QM6yz2W8QE</span>-bflKhc5oThw', <span class="symbol">'index'</span>: <span class="symbol">'news'</span>&#125;], <span class="symbol">'type'</span>: <span class="symbol">'resource_already_exists_exception'</span>, <span class="symbol">'reason'</span>: <span class="symbol">'index</span> [news/<span class="type">QM6yz2W8QE</span>-bflKhc5oThw] already exists', <span class="symbol">'index_uuid'</span>: <span class="symbol">'QM6yz2W8QE</span>-bflKhc5oThw', <span class="symbol">'index'</span>: <span class="symbol">'news'</span>&#125;, <span class="symbol">'status'</span>: <span class="number">400</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>它提示创建失败，status 状态码是 400，错误原因是 Index 已经存在了。 注意这里我们的代码里面使用了 ignore 参数为 400，这说明如果返回结果是 400 的话，就忽略这个错误不会报错，程序不会执行抛出异常。 假如我们不加 ignore 这个参数的话：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">es</span> = <span class="function"><span class="title">Elasticsearch</span>()</span></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">es.indices.create</span>(<span class="variable">index</span>=<span class="string">'news'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再次执行就会报错了：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">raise <span class="module-access"><span class="module"><span class="identifier">HTTP_EXCEPTIONS</span>.</span></span>get(status_code, TransportError)(status_code, error_message, additional_info)</span><br><span class="line">elasticsearch.exceptions.RequestError: <span class="constructor">TransportError(400, '<span class="params">resource_already_exists_exception</span>', '<span class="params">index</span> [<span class="params">news</span><span class="operator">/</span>QM6yz2W8QE-<span class="params">bflKhc5oThw</span>] <span class="params">already</span> <span class="params">exists</span>')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样程序的执行就会出现问题，所以说，我们需要善用 ignore 参数，把一些意外情况排除，这样可以保证程序的正常执行而不会中断。</p>
                  <h3 id="删除-Index"><a href="#删除-Index" class="headerlink" title="删除 Index"></a>删除 Index</h3>
                  <p>删除 Index 也是类似的，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">result = es.indices.delete(<span class="attribute">index</span>=<span class="string">'news'</span>, ignore=[400, 404])</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里也是使用了 ignore 参数，来忽略 Index 不存在而删除失败导致程序中断的问题。 如果删除成功，会输出如下结果：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="symbol">'acknowledged</span>': <span class="literal">True</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果 Index 已经被删除，再执行删除则会输出如下结果：</p>
                  <figure class="highlight sml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="symbol">'error'</span>: &#123;<span class="symbol">'root_cause'</span>: [&#123;<span class="symbol">'type'</span>: <span class="symbol">'index_not_found_exception'</span>, <span class="symbol">'reason'</span>: <span class="symbol">'no</span> such index', <span class="symbol">'resource</span>.type': <span class="symbol">'index_or_alias'</span>, <span class="symbol">'resource</span>.id': <span class="symbol">'news'</span>, <span class="symbol">'index_uuid'</span>: <span class="symbol">'_na_'</span>, <span class="symbol">'index'</span>: <span class="symbol">'news'</span>&#125;], <span class="symbol">'type'</span>: <span class="symbol">'index_not_found_exception'</span>, <span class="symbol">'reason'</span>: <span class="symbol">'no</span> such index', <span class="symbol">'resource</span>.type': <span class="symbol">'index_or_alias'</span>, <span class="symbol">'resource</span>.id': <span class="symbol">'news'</span>, <span class="symbol">'index_uuid'</span>: <span class="symbol">'_na_'</span>, <span class="symbol">'index'</span>: <span class="symbol">'news'</span>&#125;, <span class="symbol">'status'</span>: <span class="number">404</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个结果表明当前 Index 不存在，删除失败，返回的结果同样是 JSON，状态码是 400，但是由于我们添加了 ignore 参数，忽略了 400 状态码，因此程序正常执行输出 JSON 结果，而不是抛出异常。</p>
                  <h3 id="插入数据"><a href="#插入数据" class="headerlink" title="插入数据"></a>插入数据</h3>
                  <p>Elasticsearch 就像 MongoDB 一样，在插入数据的时候可以直接插入结构化字典数据，插入数据可以调用 create() 方法，例如这里我们插入一条新闻数据：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">es.indices.create(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">ignore</span>=400)</span><br><span class="line"></span><br><span class="line">data = &#123;<span class="string">'title'</span>: <span class="string">'美国留给伊拉克的是个烂摊子吗'</span>, <span class="string">'url'</span>: <span class="string">'http://view.news.qq.com/zt2011/usa_iraq/index.htm'</span>&#125;</span><br><span class="line">result = es.create(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">id</span>=1, <span class="attribute">body</span>=data)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先声明了一条新闻数据，包括标题和链接，然后通过调用 create() 方法插入了这条数据，在调用 create() 方法时，我们传入了四个参数，index 参数代表了索引名称，doc_type 代表了文档类型，body 则代表了文档具体内容，id 则是数据的唯一标识 ID。 运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'_index': 'news', '_type': 'politics', '_id': '1', '_version': <span class="number">1</span>, 'result': 'created', '_shards': &#123;'total': <span class="number">2</span>, 'successful': <span class="number">1</span>, 'failed': <span class="number">0</span>&#125;, '_seq_no': <span class="number">0</span>, '_primary_term': <span class="number">1</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果中 result 字段为 created，代表该数据插入成功。 另外其实我们也可以使用 index() 方法来插入数据，但与 create() 不同的是，create() 方法需要我们指定 id 字段来唯一标识该条数据，而 index() 方法则不需要，如果不指定 id，会自动生成一个 id，调用 index() 方法的写法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">es.index(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">body</span>=data)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>create() 方法内部其实也是调用了 index() 方法，是对 index() 方法的封装。</p>
                  <h3 id="更新数据"><a href="#更新数据" class="headerlink" title="更新数据"></a>更新数据</h3>
                  <p>更新数据也非常简单，我们同样需要指定数据的 id 和内容，调用 update() 方法即可，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'title'</span>: <span class="string">'美国留给伊拉克的是个烂摊子吗'</span>,</span><br><span class="line">    <span class="string">'url'</span>: <span class="string">'http://view.news.qq.com/zt2011/usa_iraq/index.htm'</span>,</span><br><span class="line">    <span class="string">'date'</span>: <span class="string">'2011-12-16'</span></span><br><span class="line">&#125;</span><br><span class="line">result = es.update(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">body</span>=data, <span class="attribute">id</span>=1)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们为数据增加了一个日期字段，然后调用了 update() 方法，结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'_index': 'news', '_type': 'politics', '_id': '1', '_version': <span class="number">2</span>, 'result': 'updated', '_shards': &#123;'total': <span class="number">2</span>, 'successful': <span class="number">1</span>, 'failed': <span class="number">0</span>&#125;, '_seq_no': <span class="number">1</span>, '_primary_term': <span class="number">1</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到返回结果中，result 字段为 updated，即表示更新成功，另外我们还注意到有一个字段 _version，这代表更新后的版本号数，2 代表这是第二个版本，因为之前已经插入过一次数据，所以第一次插入的数据是版本 1，可以参见上例的运行结果，这次更新之后版本号就变成了 2，以后每更新一次，版本号都会加 1。 另外更新操作其实利用 index() 方法同样可以做到，写法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">es.index(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">body</span>=data, <span class="attribute">id</span>=1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，index() 方法可以代替我们完成两个操作，如果数据不存在，那就执行插入操作，如果已经存在，那就执行更新操作，非常方便。</p>
                  <h3 id="删除数据"><a href="#删除数据" class="headerlink" title="删除数据"></a>删除数据</h3>
                  <p>如果想删除一条数据可以调用 delete() 方法，指定需要删除的数据 id 即可，写法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">result = es.delete(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">id</span>=1)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'_index': 'news', '_type': 'politics', '_id': '1', '_version': <span class="number">3</span>, 'result': 'deleted', '_shards': &#123;'total': <span class="number">2</span>, 'successful': <span class="number">1</span>, 'failed': <span class="number">0</span>&#125;, '_seq_no': <span class="number">2</span>, '_primary_term': <span class="number">1</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到运行结果中 result 字段为 deleted，代表删除成功，_version 变成了 3，又增加了 1。</p>
                  <h3 id="查询数据"><a href="#查询数据" class="headerlink" title="查询数据"></a>查询数据</h3>
                  <p>上面的几个操作都是非常简单的操作，普通的数据库如 MongoDB 都是可以完成的，看起来并没有什么了不起的，Elasticsearch 更特殊的地方在于其异常强大的检索功能。 对于中文来说，我们需要安装一个分词插件，这里使用的是 elasticsearch-analysis-ik，GitHub 链接为：<a href="https://github.com/medcl/elasticsearch-analysis-ik" target="_blank" rel="noopener">https://github.com/medcl/elasticsearch-analysis-ik</a>，这里我们使用 Elasticsearch 的另一个命令行工具 elasticsearch-plugin 来安装，这里安装的版本是 6.2.4，请确保和 Elasticsearch 的版本对应起来，命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">elasticsearch-plugin install https:<span class="regexp">//gi</span>thub.com<span class="regexp">/medcl/</span>elasticsearch-analysis-ik<span class="regexp">/releases/</span>download<span class="regexp">/v6.2.4/</span>elasticsearch-analysis-ik-<span class="number">6.2</span>.<span class="number">4</span>.zip</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的版本号请替换成你的 Elasticsearch 的版本号。 安装之后重新启动 Elasticsearch 就可以了，它会自动加载安装好的插件。 首先我们新建一个索引并指定需要分词的字段，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> elasticsearch import Elasticsearch</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">mapping = &#123;</span><br><span class="line">    <span class="string">'properties'</span>: &#123;</span><br><span class="line">        <span class="string">'title'</span>: &#123;</span><br><span class="line">            <span class="string">'type'</span>: <span class="string">'text'</span>,</span><br><span class="line">            <span class="string">'analyzer'</span>: <span class="string">'ik_max_word'</span>,</span><br><span class="line">            <span class="string">'search_analyzer'</span>: <span class="string">'ik_max_word'</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line">es.indices.delete(<span class="attribute">index</span>=<span class="string">'news'</span>, ignore=[400, 404])</span><br><span class="line">es.indices.create(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">ignore</span>=400)</span><br><span class="line">result = es.indices.put_mapping(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">body</span>=mapping)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们先将之前的索引删除了，然后新建了一个索引，然后更新了它的 mapping 信息，mapping 信息中指定了分词的字段，指定了字段的类型 type 为 text，分词器 analyzer 和 搜索分词器 search_analyzer 为 ik_max_word，即使用我们刚才安装的中文分词插件。如果不指定的话则使用默认的英文分词器。 接下来我们插入几条新的数据：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">datas = [</span><br><span class="line">    &#123;</span><br><span class="line">        'title': '美国留给伊拉克的是个烂摊子吗',</span><br><span class="line">        'url': 'http://view.news.qq.com/zt<span class="number">2011</span>/usa_iraq/index.htm',</span><br><span class="line">        'date': '<span class="number">2011-12-16</span>'</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">        'title': '公安部：各地校车将享最高路权',</span><br><span class="line">        'url': 'http://www.chinanews.com/gn/<span class="number">2011/12-16/353607</span>7.shtml',</span><br><span class="line">        'date': '<span class="number">2011-12-16</span>'</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">        'title': '中韩渔警冲突调查：韩警平均每天扣1艘中国渔船',</span><br><span class="line">        'url': 'https://news.qq.com/a/<span class="number">20111216/001044</span>.htm',</span><br><span class="line">        'date': '<span class="number">2011-12-17</span>'</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">        'title': '中国驻洛杉矶领事馆遭亚裔男子枪击 嫌犯已自首',</span><br><span class="line">        'url': 'http://news.ifeng.com/world/detail_<span class="number">2011</span>_12/16/<span class="number">11372558</span>_0.shtml',</span><br><span class="line">        'date': '<span class="number">2011-12-18</span>'</span><br><span class="line">    &#125;</span><br><span class="line">]</span><br><span class="line"></span><br><span class="line">for data in datas:</span><br><span class="line">    es.index(index='news', doc_type='politics', body=data)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们指定了四条数据，都带有 title、url、date 字段，然后通过 index() 方法将其插入 Elasticsearch 中，索引名称为 news，类型为 politics。 接下来我们根据关键词查询一下相关内容：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = es.search(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到查询出了所有插入的四条数据：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"took"</span>: <span class="number">0</span>,</span><br><span class="line">  <span class="attr">"timed_out"</span>: <span class="literal">false</span>,</span><br><span class="line">  <span class="attr">"_shards"</span>: &#123;</span><br><span class="line">    <span class="attr">"total"</span>: <span class="number">5</span>,</span><br><span class="line">    <span class="attr">"successful"</span>: <span class="number">5</span>,</span><br><span class="line">    <span class="attr">"skipped"</span>: <span class="number">0</span>,</span><br><span class="line">    <span class="attr">"failed"</span>: <span class="number">0</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"hits"</span>: &#123;</span><br><span class="line">    <span class="attr">"total"</span>: <span class="number">4</span>,</span><br><span class="line">    <span class="attr">"max_score"</span>: <span class="number">1.0</span>,</span><br><span class="line">    <span class="attr">"hits"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"c05G9mQBD9BuE5fdHOUT"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">1.0</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"美国留给伊拉克的是个烂摊子吗"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"http://view.news.qq.com/zt2011/usa_iraq/index.htm"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-16"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;,</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"dk5G9mQBD9BuE5fdHOUm"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">1.0</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"中国驻洛杉矶领事馆遭亚裔男子枪击，嫌犯已自首"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"http://news.ifeng.com/world/detail_2011_12/16/11372558_0.shtml"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-18"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;,</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"dU5G9mQBD9BuE5fdHOUj"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">1.0</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"中韩渔警冲突调查：韩警平均每天扣1艘中国渔船"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"https://news.qq.com/a/20111216/001044.htm"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-17"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;,</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"dE5G9mQBD9BuE5fdHOUf"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">1.0</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"公安部：各地校车将享最高路权"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"http://www.chinanews.com/gn/2011/12-16/3536077.shtml"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-16"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;</span><br><span class="line">    ]</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到返回结果会出现在 hits 字段里面，然后其中有 total 字段标明了查询的结果条目数，还有 max_score 代表了最大匹配分数。 另外我们还可以进行全文检索，这才是体现 Elasticsearch 搜索引擎特性的地方：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">dsl = &#123;</span><br><span class="line">    <span class="string">'query'</span>: &#123;</span><br><span class="line">        <span class="string">'match'</span>: &#123;</span><br><span class="line">            <span class="string">'title'</span>: <span class="string">'中国 领事馆'</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">es = Elasticsearch()</span><br><span class="line">result = es.search(<span class="attribute">index</span>=<span class="string">'news'</span>, <span class="attribute">doc_type</span>=<span class="string">'politics'</span>, <span class="attribute">body</span>=dsl)</span><br><span class="line"><span class="builtin-name">print</span>(json.dumps(result, <span class="attribute">indent</span>=2, <span class="attribute">ensure_ascii</span>=<span class="literal">False</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用 Elasticsearch 支持的 DSL 语句来进行查询，使用 match 指定全文检索，检索的字段是 title，内容是“中国领事馆”，搜索结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"took"</span>: <span class="number">1</span>,</span><br><span class="line">  <span class="attr">"timed_out"</span>: <span class="literal">false</span>,</span><br><span class="line">  <span class="attr">"_shards"</span>: &#123;</span><br><span class="line">    <span class="attr">"total"</span>: <span class="number">5</span>,</span><br><span class="line">    <span class="attr">"successful"</span>: <span class="number">5</span>,</span><br><span class="line">    <span class="attr">"skipped"</span>: <span class="number">0</span>,</span><br><span class="line">    <span class="attr">"failed"</span>: <span class="number">0</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"hits"</span>: &#123;</span><br><span class="line">    <span class="attr">"total"</span>: <span class="number">2</span>,</span><br><span class="line">    <span class="attr">"max_score"</span>: <span class="number">2.546152</span>,</span><br><span class="line">    <span class="attr">"hits"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"dk5G9mQBD9BuE5fdHOUm"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">2.546152</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"中国驻洛杉矶领事馆遭亚裔男子枪击，嫌犯已自首"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"http://news.ifeng.com/world/detail_2011_12/16/11372558_0.shtml"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-18"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;,</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">"_index"</span>: <span class="string">"news"</span>,</span><br><span class="line">        <span class="attr">"_type"</span>: <span class="string">"politics"</span>,</span><br><span class="line">        <span class="attr">"_id"</span>: <span class="string">"dU5G9mQBD9BuE5fdHOUj"</span>,</span><br><span class="line">        <span class="attr">"_score"</span>: <span class="number">0.2876821</span>,</span><br><span class="line">        <span class="attr">"_source"</span>: &#123;</span><br><span class="line">          <span class="attr">"title"</span>: <span class="string">"中韩渔警冲突调查：韩警平均每天扣1艘中国渔船"</span>,</span><br><span class="line">          <span class="attr">"url"</span>: <span class="string">"https://news.qq.com/a/20111216/001044.htm"</span>,</span><br><span class="line">          <span class="attr">"date"</span>: <span class="string">"2011-12-17"</span></span><br><span class="line">        &#125;</span><br><span class="line">      &#125;</span><br><span class="line">    ]</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们看到匹配的结果有两条，第一条的分数为 2.54，第二条的分数为 0.28，这是因为第一条匹配的数据中含有“中国”和“领事馆”两个词，第二条匹配的数据中不包含“领事馆”，但是包含了“中国”这个词，所以也被检索出来了，但是分数比较低。 因此可以看出，检索时会对对应的字段全文检索，结果还会按照检索关键词的相关性进行排序，这就是一个基本的搜索引擎雏形。 另外 Elasticsearch 还支持非常多的查询方式，详情可以参考官方文档：<a href="https://www.elastic.co/guide/en/elasticsearch/reference/6.3/query-dsl.html" target="_blank" rel="noopener">https://www.elastic.co/guide/en/elasticsearch/reference/6.3/query-dsl.html</a> 以上便是对 Elasticsearch 的基本介绍以及 Python 操作 Elasticsearch 的基本用法，但这仅仅是 Elasticsearch 的基本功能，它还有更多强大的功能等待着我们的探索，后面会继续更新，敬请期待。 本节代码：<a href="https://github.com/Germey/ElasticSearch" target="_blank" rel="noopener">https://github.com/Germey/ElasticSearch</a>。</p>
                  <h2 id="资料推荐"><a href="#资料推荐" class="headerlink" title="资料推荐"></a>资料推荐</h2>
                  <p>另外推荐几个不错的学习站点：</p>
                  <ul>
                    <li>Elasticsearch 权威指南：<a href="https://es.xiaoleilu.com/index.html" target="_blank" rel="noopener">https://es.xiaoleilu.com/index.html</a></li>
                    <li>全文搜索引擎 Elasticsearch 入门教程：<a href="http://www.ruanyifeng.com/blog/2017/08/elasticsearch.html" target="_blank" rel="noopener">http://www.ruanyifeng.com/blog/2017/08/elasticsearch.html</a></li>
                    <li>Elastic 中文社区：<a href="https://www.elasticsearch.cn/" target="_blank" rel="noopener">https://www.elasticsearch.cn/</a></li>
                  </ul>
                  <h2 id="参考资料"><a href="#参考资料" class="headerlink" title="参考资料"></a>参考资料</h2>
                  <ul>
                    <li><a href="https://es.xiaoleilu.com/index.html" target="_blank" rel="noopener">https://es.xiaoleilu.com/index.html</a></li>
                    <li><a href="https://blog.csdn.net/y472360651/article/details/76468327" target="_blank" rel="noopener">https://blog.csdn.net/y472360651/article/details/76468327</a></li>
                    <li><a href="https://elasticsearch-py.readthedocs.io/en/master/" target="_blank" rel="noopener">https://elasticsearch-py.readthedocs.io/en/master/</a></li>
                    <li><a href="https://es.xiaoleilu.com/010_Intro/10_Installing_ES.html" target="_blank" rel="noopener">https://es.xiaoleilu.com/010_Intro/10_Installing_ES.html</a></li>
                    <li><a href="https://github.com/medcl/elasticsearch-analysis-ik" target="_blank" rel="noopener">https://github.com/medcl/elasticsearch-analysis-ik</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-08-02 01:14:22" itemprop="dateCreated datePublished" datetime="2018-08-02T01:14:22+08:00">2018-08-02</time>
                </span>
                <span id="/6214.html" class="post-meta-item leancloud_visitors" data-flag-title="Elasticsearch 基本介绍及其与 Python 的对接实现" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>11 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6182.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6182.html" class="post-title-link" itemprop="url">Python glom包初探</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="大家好，-我不是崔老师，我是四毛，下面是我的个人公众号，欢迎大家关注。"><a href="#大家好，-我不是崔老师，我是四毛，下面是我的个人公众号，欢迎大家关注。" class="headerlink" title="大家好， 我不是崔老师，我是四毛，下面是我的个人公众号，欢迎大家关注。"></a>大家好， 我不是崔老师，我是四毛，下面是我的个人公众号，欢迎大家关注。</h1>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/201802111155445124jhQyer.yasuotu.gif" alt=""></p>
                  <h1 id="好久没有写东西了，一直都记录在了自己的笔记上，这一篇是关于glom的一个介绍与初步使用，后期会将里面的各种API再给大家介绍下，同时，最近在搞爬虫的实时数据监控，也挺有意思，后面会和大家分享，敬请期待。"><a href="#好久没有写东西了，一直都记录在了自己的笔记上，这一篇是关于glom的一个介绍与初步使用，后期会将里面的各种API再给大家介绍下，同时，最近在搞爬虫的实时数据监控，也挺有意思，后面会和大家分享，敬请期待。" class="headerlink" title="好久没有写东西了，一直都记录在了自己的笔记上，这一篇是关于glom的一个介绍与初步使用，后期会将里面的各种API再给大家介绍下，同时，最近在搞爬虫的实时数据监控，也挺有意思，后面会和大家分享，敬请期待。"></a>好久没有写东西了，一直都记录在了自己的笔记上，这一篇是关于glom的一个介绍与初步使用，后期会将里面的各种API再给大家介绍下，同时，最近在搞爬虫的实时数据监控，也挺有意思，后面会和大家分享，敬请期待。</h1>
                  <p>猛然发现，英语水平巅峰就在高考那一天。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/吃惊表情1.jpg" alt=""> 因为是边看，边练习，然后翻译，所以个人理解可能有偏差，有错误的地方，请大家指正。 首先，这个库是用来处理一些嵌套的数据的，作者也在PyCon 2018上做了个分享，老美的PyCon还是有点质量的，不像国内的，搞的什么玩意。 视频地址：<a href="https://www.youtube.com/watch?v=bTAFl8P2DkE&amp;t=18m07s" target="_blank" rel="noopener">https://www.youtube.com/watch?v=bTAFl8P2DkE&amp;t=18m07s</a> </p>
                  <h2 id="更新-2018年7月28日10-32-08"><a href="#更新-2018年7月28日10-32-08" class="headerlink" title="更新: 2018年7月28日10:32:08"></a>更新: 2018年7月28日10:32:08</h2>
                  <p>经过咨询库的作者，在最后留的那个问题的准确解法如下：</p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> glom</span><br><span class="line"></span><br><span class="line">target = &#123;</span><br><span class="line">    <span class="string">'data'</span>: &#123;</span><br><span class="line">        <span class="string">'name'</span>: <span class="string">'just_test'</span>,</span><br><span class="line">        <span class="string">'likes'</span>: [&#123;<span class="string">'ball'</span>: <span class="string">'basketball'</span>&#125;,</span><br><span class="line">                  &#123;<span class="string">'ball'</span>: <span class="string">'football'</span>&#125;,</span><br><span class="line">                  &#123;<span class="string">'water'</span>: <span class="string">'swim'</span>&#125;]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">spec = &#123;</span><br><span class="line">    <span class="string">'name'</span> : (<span class="string">'data.name'</span>),</span><br><span class="line">    <span class="string">'likes'</span> : (<span class="string">'data'</span>, <span class="string">'likes'</span>, [glom.Coalesce(<span class="string">'ball'</span>, <span class="string">'water'</span>)])</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">print glom.glom(target, spec)</span><br><span class="line">####</span><br><span class="line">&#123;<span class="string">'name'</span>: <span class="string">'just_test'</span>, <span class="string">'likes'</span>: [<span class="string">'basketball'</span>, <span class="string">'football'</span>, <span class="string">'swim'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>非常棒，准确来说就是得灵活运用Coalesce方法啊，不能太死板。非常Pythonic。 另附网址，作者有个很搞笑little four hair ,哈哈哈哈 <a href="https://github.com/mahmoud/glom/issues/46" target="_blank" rel="noopener">Issue地址</a> </p>
                  <h1 id="1-官方文档地址"><a href="#1-官方文档地址" class="headerlink" title="1. 官方文档地址"></a>1. 官方文档地址</h1>
                  <p><a href="https://glom.readthedocs.io/en/latest/tutorial.html" target="_blank" rel="noopener">文档地址</a></p>
                  <h1 id="2-安装方法"><a href="#2-安装方法" class="headerlink" title="2. 安装方法"></a>2. 安装方法</h1>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> glom</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h1 id="3-正式开始"><a href="#3-正式开始" class="headerlink" title="3. 正式开始"></a>3. 正式开始</h1>
                  <p>glom，官方的说法是用PYTHONIC的方式来处理内嵌的数据。对于现实世界中的数据处理更加给力，现实世界中的数据，我的理解就是AJAX越来越流行了，处理这类数据会越来越频繁。有如下特点：</p>
                  <ul>
                    <li>对于嵌套数据结构的基于路径式的访问</li>
                    <li>可读，有意义的错误消息</li>
                    <li>声明性数据转换，使用轻量级，Pythonic规范</li>
                    <li>内置数据探索和调试功能</li>
                  </ul>
                  <h2 id="3-1-原始处理嵌套数据"><a href="#3-1-原始处理嵌套数据" class="headerlink" title="3.1 原始处理嵌套数据"></a>3.1 原始处理嵌套数据</h2>
                  <p>下面的脚本包导入</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> glom <span class="keyword">import</span> glom</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面的data就是个简单的嵌套数据，一般都可以用下面几种方法进行处理</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data = &#123;<span class="string">'a'</span>: &#123;<span class="string">'b'</span>: &#123;<span class="string">'c'</span>: <span class="string">'d'</span>&#125;&#125;&#125;</span><br><span class="line">data[<span class="string">'a'</span>][<span class="string">'b'</span>][<span class="string">'c'</span>]</span><br><span class="line">data.<span class="builtin-name">get</span>(<span class="string">'a'</span>).<span class="builtin-name">get</span>(<span class="string">'b'</span>).<span class="builtin-name">get</span>(<span class="string">'c'</span>)</span><br><span class="line">data.<span class="builtin-name">get</span>(<span class="string">'a'</span>, &#123;&#125;).<span class="builtin-name">get</span>(<span class="string">'b'</span>,&#123;&#125;).<span class="builtin-name">get</span>(<span class="string">'c'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但是当我们的数据改变成下面的这样时：</p>
                  <figure class="highlight markdown">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data2 = &#123;'a': &#123;'b': None&#125;&#125;</span><br><span class="line">data2[<span class="string">'a'</span>][<span class="symbol">'b'</span>][<span class="string">'c'</span>]</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">...</span><br><span class="line">TypeError: 'NoneType' object has no attribute '<span class="strong">__getitem__</span>'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>会报错，而且由于是嵌套数据，从错误信息里我们只知道有个None值，但是到底谁是呢，是a，是b呢，反正肯定不是我们的朋友小哪吒。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/07/4399_9165819336.jpg" alt=""></p>
                  <h2 id="3-2-glom出场"><a href="#3-2-glom出场" class="headerlink" title="3.2 glom出场"></a>3.2 glom出场</h2>
                  <p>那么glom怎么处理上面的数据呢？ 如其所言，路径式：</p>
                  <figure class="highlight powershell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">data</span> = &#123;<span class="string">'a'</span>: &#123;<span class="string">'b'</span>: &#123;<span class="string">'c'</span>: <span class="string">'d'</span>&#125;&#125;&#125;</span><br><span class="line">print glom(<span class="keyword">data</span>, <span class="string">'a.b.c'</span>)  <span class="comment"># d</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>看起来还是很优雅， 很Pythonic。</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data2 = &#123;<span class="string">'a'</span>: &#123;<span class="string">'b'</span>: None&#125;&#125;</span><br><span class="line"><span class="function"><span class="title">glom</span><span class="params">(data2, <span class="string">'a.b.c'</span>)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>错误信息如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">glom.core.<span class="string">PathAccessError:</span> could not access <span class="string">'c'</span>, part <span class="number">2</span> of Path(<span class="string">'a'</span>, <span class="string">'b'</span>, <span class="string">'c'</span>), got <span class="string">error:</span> AttributeError(<span class="string">"'NoneType' object has no attribute 'c'"</span>,)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>很明显，这个错误就很直观。 难道仅仅只有这个？当然不是</p>
                  <h3 id="3-2-1-Going-Beyond-Access"><a href="#3-2-1-Going-Beyond-Access" class="headerlink" title="3.2.1  Going Beyond Access"></a>3.2.1 Going Beyond Access</h3>
                  <p>上面的是原标题，我的理解是不仅仅获取数据，还有别的呢。 首先，介绍两个基本的术语</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">target</span> <span class="string">目标数据，可以是字典，列表，或其他任意的对象</span></span><br><span class="line"><span class="attr">spec</span>  <span class="string">我们想要的输出格式 【specifications】， 定义你自己所需要的格式</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在让我们跟随宇航员的脚步，探索太阳系吧。</p>
                  <ul>
                    <li>获取某个行星的名字：</li>
                  </ul>
                  <figure class="highlight vala">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">target = &#123;<span class="string">'galaxy'</span>: &#123;<span class="string">'system'</span>: &#123;<span class="string">'planet'</span>: <span class="string">'jupiter'</span>&#125;&#125;&#125;</span><br><span class="line"><span class="meta"># 这个格式就是需要个字段值，所以输出的就是个字段值</span></span><br><span class="line">spec = <span class="string">'galaxy.system.planet'</span></span><br><span class="line">glom(target, spec)</span><br><span class="line"><span class="meta"># 'jupyter'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>现在，宇航员们想把行星的名字放进一个列表中，数据是这样：</li>
                  </ul>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">target</span> = &#123;<span class="string">'system'</span>: &#123;<span class="string">'planets'</span>: [&#123;<span class="string">'name'</span>: <span class="string">'earth'</span>&#125;, &#123;<span class="string">'name'</span>: <span class="string">'jupiter'</span>&#125;]&#125;&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>通常，处理这样的话，都要写个循环，或者搞个列表解析式，那么glom怎么处理呢？</li>
                  </ul>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">glom(target, (<span class="string">'system.planets'</span>, [<span class="string">'name'</span>]))</span><br><span class="line">print glom(target, spec)</span><br><span class="line"># [<span class="string">'earth'</span>, <span class="string">'jupiter'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>是不是很简单。那么现在新需求又来了，宇航员想得到下面这个数据里面的行星的卫星的数:</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">target</span> = &#123;<span class="string">'system'</span>: &#123;<span class="string">'planets'</span>: [&#123;<span class="string">'name'</span>: <span class="string">'earth'</span>, <span class="string">'moons'</span>: <span class="number">1</span>&#125;,</span><br><span class="line">                                  &#123;<span class="string">'name'</span>: <span class="string">'jupiter'</span>, <span class="string">'moons'</span>: <span class="number">69</span>&#125;]&#125;&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>glom解决方法：</li>
                  </ul>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># 自定义的格式</span><br><span class="line">spec = &#123;<span class="string">'names'</span>: (<span class="string">'system.planets'</span>, [<span class="string">'name'</span>]),</span><br><span class="line">        <span class="string">'moons'</span>: (<span class="string">'system.planets'</span>, [<span class="string">'moons'</span>])&#125;</span><br><span class="line">print glom(target, spec)</span><br><span class="line"># &#123;<span class="string">'moons'</span>: [<span class="number">1</span>, <span class="number">69</span>], <span class="string">'names'</span>: [<span class="string">'earth'</span>, <span class="string">'jupiter'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="3-2-2-Changing-Requirements"><a href="#3-2-2-Changing-Requirements" class="headerlink" title="3.2.2  Changing Requirements"></a>3.2.2 Changing Requirements</h3>
                  <p>Coalesce 是glom定义的一种结构，允许我们对于spec中的子spec进行进一步的处理，你只要在子spec中将可能存在的值定义好就行了，听起来有点绕，现在来梳理一下。</p>
                  <ul>
                    <li>首先，子spec是什么？</li>
                  </ul>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">spec = &#123;<span class="string">'names'</span>: (<span class="string">'system.planets'</span>, [<span class="string">'name'</span>]),</span><br><span class="line">         <span class="string">'moons'</span>: (<span class="string">'system.planets'</span>, [<span class="string">'moons'</span>])&#125;</span><br><span class="line"># 以这个为例，这里面的<span class="keyword">system</span>.planets就是个子spec</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>然后，使用其解析数据：</li>
                  </ul>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">target = &#123;<span class="string">'system'</span>: &#123;</span><br><span class="line">    <span class="string">'planets'</span>: [&#123;<span class="string">'name'</span>: <span class="string">'earth'</span>, <span class="string">'moons'</span>: <span class="number">1</span>&#125;, &#123;<span class="string">'name'</span>: <span class="string">'jupiter'</span>, <span class="string">'moons'</span>: <span class="number">69</span>&#125;],</span><br><span class="line">&#125;</span><br><span class="line">&#125;</span><br><span class="line">spec = &#123;<span class="string">'names'</span>: (<span class="symbol">Coalesce</span>(<span class="string">'system.planets'</span>, <span class="string">'system.dwarf_planets'</span>), [<span class="string">'name'</span>]),</span><br><span class="line">         <span class="string">'moons'</span>: (<span class="symbol">Coalesce</span>(<span class="string">'system.planets'</span>, <span class="string">'system.dwarf_planets'</span>), [<span class="string">'moons'</span>])&#125;</span><br><span class="line"></span><br><span class="line">print glom(target, spec)</span><br><span class="line"># &#123;<span class="string">'moons'</span>: [<span class="number">1</span>, <span class="number">69</span>], <span class="string">'names'</span>: [<span class="string">'earth'</span>, <span class="string">'jupiter'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>接着当我们的数据变成了这个以后</li>
                  </ul>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">target = &#123;<span class="string">'system'</span>: &#123;<span class="string">'dwarf_planets'</span>: [&#123;<span class="string">'name'</span>: <span class="string">'pluto'</span>, <span class="string">'moons'</span>: <span class="number">5</span>&#125;,</span><br><span class="line">                                        &#123;<span class="string">'name'</span>: <span class="string">'ceres'</span>, <span class="string">'moons'</span>: <span class="number">0</span>&#125;]&#125;&#125;</span><br><span class="line">spec = &#123;<span class="string">'names'</span>: (<span class="symbol">Coalesce</span>(<span class="string">'system.planets'</span>, <span class="string">'system.dwarf_planets'</span>), [<span class="string">'name'</span>]),</span><br><span class="line">         <span class="string">'moons'</span>: (<span class="symbol">Coalesce</span>(<span class="string">'system.planets'</span>, <span class="string">'system.dwarf_planets'</span>), [<span class="string">'moons'</span>])&#125;</span><br><span class="line">print glom(target, spec)</span><br><span class="line"># &#123;<span class="string">'moons'</span>: [<span class="number">5</span>, <span class="number">0</span>], <span class="string">'names'</span>: [<span class="string">'pluto'</span>, <span class="string">'ceres'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，依然可以使用相同的spec来解析不同的目标数据。 有意思的是，你可以在target里面同时写入plantes和dwarf_plants数据试试看，会返回什么数据。 【这里应该是个惰性的匹配，只要匹配到一个，后面的就不再去匹配了】</p>
                  <h3 id="3-2-3-True-Python-Native"><a href="#3-2-3-True-Python-Native" class="headerlink" title="3.2.3  True Python Native"></a>3.2.3 True Python Native</h3>
                  <p>真正的原生python 在glom里面，你可以传值给python里面的任意的函数 举例：</p>
                  <ul>
                    <li>求和</li>
                  </ul>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">target = &#123;<span class="string">'system'</span>: &#123;<span class="string">'planets'</span>: [&#123;<span class="string">'name'</span>: <span class="string">'earth'</span>, <span class="string">'moons'</span>: <span class="number">1</span>&#125;,</span><br><span class="line">                                  &#123;<span class="string">'name'</span>: <span class="string">'jupiter'</span>, <span class="string">'moons'</span>: <span class="number">69</span>&#125;]&#125;&#125;</span><br><span class="line"></span><br><span class="line">print glom(target, &#123;<span class="string">'moon_count'</span>: (<span class="string">'system.planets'</span>, [<span class="string">'moons'</span>], sum)&#125;)</span><br><span class="line"></span><br><span class="line"># &#123;<span class="string">'moon_count'</span>: <span class="number">70</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>原教程这里还有个案例，但是我还没有理解好，就不写出来了，大家可以点击链接自己看一下。 </p>
                  <h1 id="4-结论"><a href="#4-结论" class="headerlink" title="4. 结论"></a>4. 结论</h1>
                  <p>下一节，为大家带来其中一些重要的函数。 最后，在用的过程中，一直有个疑问，数据如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">target</span> = &#123;</span><br><span class="line">    <span class="string">'data'</span>: &#123;</span><br><span class="line">        <span class="string">'name'</span>: <span class="string">'just_test'</span>,</span><br><span class="line">        <span class="string">'likes'</span>: [&#123;<span class="string">'ball'</span>: <span class="string">'basketball'</span>&#125;,</span><br><span class="line">                  &#123;<span class="string">'ball'</span>: <span class="string">'football'</span>&#125;,</span><br><span class="line">                  &#123;<span class="string">'water'</span>: <span class="string">'swim'</span>&#125;]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在，我想返回的数据格式为：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="string">'name'</span>: <span class="string">'just_for_test'</span>, <span class="string">'likes'</span>: [<span class="string">'basketball'</span>, <span class="string">'football'</span>, <span class="string">'water'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>一开始我以为可以这么用：</p>
                  <figure class="highlight sml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">spec = &#123;</span><br><span class="line">    <span class="symbol">'name'</span>: (<span class="symbol">'data</span>.name'),</span><br><span class="line">    <span class="symbol">'likes'</span>: (<span class="symbol">'data</span>.likes', [<span class="symbol">'ball'</span>, <span class="symbol">'water'</span>] ),</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但是不行，这样会报错。后来用了另外的方法：</p>
                  <figure class="highlight gml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">spec = &#123;</span><br><span class="line">    <span class="string">'name'</span>: (<span class="string">'data.name'</span>),</span><br><span class="line">    <span class="string">'likes'</span>: (<span class="string">'data.likes'</span>, [lambda <span class="symbol">x</span>: <span class="symbol">x</span>.values()[<span class="number">0</span>] <span class="keyword">if</span> <span class="string">'ball'</span> <span class="keyword">or</span> <span class="string">'water'</span> in <span class="symbol">x</span>.keys() <span class="keyword">else</span> <span class="string">''</span>] ),</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">print glom(target, spec)</span><br><span class="line"># &#123;<span class="string">'name'</span>: <span class="string">'just_test'</span>, <span class="string">'likes'</span>: [<span class="string">'basketball'</span>, <span class="string">'football'</span>, <span class="string">'swim'</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样感觉很不爽啊，还望会的同学不吝赐教啊。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-07-28 18:19:35" itemprop="dateCreated datePublished" datetime="2018-07-28T18:19:35+08:00">2018-07-28</time>
                </span>
                <span id="/6182.html" class="post-meta-item leancloud_visitors" data-flag-title="Python glom包初探" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6173.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6173.html" class="post-title-link" itemprop="url">《Python3网络爬虫开发实战》第三波赠书活动来了！</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="开门见山"><a href="#开门见山" class="headerlink" title="开门见山"></a>开门见山</h2>
                  <p>话不多说了！第三波送书活动来了！这次送 20 本签名版《Python3网络爬虫开发实战》。 本书目前上市三个月已经重印 6 次，上市三个月以来长期位居京东计算机类新书榜第一位（现已不算新书），目前在豆瓣的评分是 9.2 分。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/05/Python-3网格爬虫开发实战-立体图-857x1100.jpg" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG1030.jpeg" alt=""></p>
                  <h2 id="书籍介绍"><a href="#书籍介绍" class="headerlink" title="书籍介绍"></a><strong>书籍介绍</strong></h2>
                  <p>本书<strong>《Python3网络爬虫开发实战》</strong>全面介绍了利用 Python3 开发网络爬虫的知识，书中首先详细介绍了各种类型的环境配置过程和爬虫基础知识，还讨论了 urllib、requests 等请求库和 Beautiful Soup、XPath、pyquery 等解析库以及文本和各类数据库的存储方法，另外本书通过多个真实新鲜案例介绍了分析 Ajax 进行数据爬取，Selenium 和 Splash 进行动态网站爬取的过程，接着又分享了一些切实可行的爬虫技巧，比如使用代理爬取和维护动态代理池的方法、ADSL 拨号代理的使用、各类验证码（图形、极验、点触、宫格等）的破解方法、模拟登录网站爬取的方法及 Cookies 池的维护等等。 此外，本书的内容还远远不止这些，作者还结合移动互联网的特点探讨了使用 Charles、mitmdump、Appium 等多种工具实现 App 抓包分析、加密参数接口爬取、微信朋友圈爬取的方法。此外本书还详细介绍了 pyspider 框架、Scrapy 框架的使用和分布式爬虫的知识，另外对于优化及部署工作，本书还包括 Bloom Filter 效率优化、Docker 和 Scrapyd 爬虫部署、分布式爬虫管理框架Gerapy 的分享。 全书共 604 页，足足两斤重呢~ 定价为 99 元！</p>
                  <h2 id="作者介绍"><a href="#作者介绍" class="headerlink" title="作者介绍"></a><strong>作者介绍</strong></h2>
                  <p>看书就先看看谁写的嘛，我们来了解一下~ 崔庆才，静觅博客博主（<a href="https://cuiqingcai.com），博客" target="_blank" rel="noopener">https://cuiqingcai.com），博客</a> Python 爬虫博文阅读量已过千万，北京航空航天大学硕士，天善智能、网易云课堂讲师，微软小冰大数据工程师，有多个大型分布式爬虫项目经验，乐于技术分享，文章通俗易懂 ^<em>^ 附皂片一张 ~(@^</em>^@)~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG785-240x320.jpeg" alt=""></p>
                  <h2 id="图文介绍"><a href="#图文介绍" class="headerlink" title="图文介绍"></a><strong>图文介绍</strong></h2>
                  <p>呕心沥血设计的宣传图也得放一下~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG556.jpeg" alt=""></p>
                  <h2 id="专家评论"><a href="#专家评论" class="headerlink" title="专家评论"></a><strong>专家评论</strong></h2>
                  <p>书是好是坏，得让专家看评一评呀，那么下面就是几位专家的精彩评论，快来看看吧~ 在互联网软件开发工程师的分类中，爬虫工程师是非常重要的。爬虫工作往往是一个公司核心业务开展的基础，数据抓取下来，才有后续的加工处理和最终展现。此时数据的抓取规模、稳定性、实时性、准确性就显得非常重要。早期的互联网充分开放互联，数据获取的难度很小。随着各大公司对数据资产日益看重，反爬水平也在不断提高，各种新技术不断给爬虫软件提出新的课题。本书作者对爬虫的各个领域都有深刻研究，书中探讨了Ajax数据的抓取、动态渲染页面的抓取、验证码识别、模拟登录等高级话题，同时也结合移动互联网的特点探讨了App的抓取等。更重要的是，本书提供了大量源码，可以帮助读者更好地理解相关内容。强烈推荐给各位技术爱好者阅读！</p>
                  <p><strong>——梁斌</strong>，八友科技总经理</p>
                  <p>数据既是当今大数据分析的前提，也是各种人工智能应用场景的基础。得数据者得天下，会爬虫者走遍天下也不怕！一册在手，让小白到老司机都能有所收获！</p>
                  <p><strong>——李舟军</strong>，北京航空航天大学教授，博士生导师</p>
                  <p>本书从爬虫入门到分布式抓取，详细介绍了爬虫技术的各个要点，并针对不同的场景提出了对应的解决方案。另外，书中通过大量的实例来帮助读者更好地学习爬虫技术，通俗易懂，干货满满。强烈推荐给大家！</p>
                  <p><strong>——宋睿华</strong>，微软小冰首席科学家</p>
                  <p>有人说中国互联网的带宽全给各种爬虫占据了，这说明网络爬虫的重要性以及中国互联网数据封闭垄断的现状。爬是一种能力，爬是为了不爬。</p>
                  <p><strong>——施水才</strong>，北京拓尔思信息技术股份有限公司总裁</p>
                  <h2 id="全书目录"><a href="#全书目录" class="headerlink" title="全书目录"></a><strong>全书目录</strong></h2>
                  <p>书的目录也有~ 看这里！</p>
                  <ul>
                    <li><strong>1-开发环境配置</strong></li>
                    <li>1.1-Python3的安装</li>
                    <li>1.2-请求库的安装</li>
                    <li>1.3-解析库的安装</li>
                    <li>1.4-数据库的安装</li>
                    <li>1.5-存储库的安装</li>
                    <li>1.6-Web库的安装</li>
                    <li>1.7-App爬取相关库的安装</li>
                    <li>1.8-爬虫框架的安装</li>
                    <li>1.9-部署相关库的安装</li>
                    <li><strong>2-爬虫基础</strong></li>
                    <li>2.1-HTTP基本原理</li>
                    <li>2.2-网页基础</li>
                    <li>2.3-爬虫的基本原理</li>
                    <li>2.4-会话和Cookies</li>
                    <li>2.5-代理的基本原理</li>
                    <li><strong>3-基本库的使用</strong></li>
                    <li>3.1-使用urllib</li>
                    <li>3.1.1-发送请求</li>
                    <li>3.1.2-处理异常</li>
                    <li>3.1.3-解析链接</li>
                    <li>3.1.4-分析Robots协议</li>
                    <li>3.2-使用requests</li>
                    <li>3.2.1-基本用法</li>
                    <li>3.2.2-高级用法</li>
                    <li>3.3-正则表达式</li>
                    <li>3.4-抓取猫眼电影排行</li>
                    <li><strong>4-解析库的使用</strong></li>
                    <li>4.1-使用XPath</li>
                    <li>4.2-使用Beautiful Soup</li>
                    <li>4.3-使用pyquery</li>
                    <li><strong>5-数据存储</strong></li>
                    <li>5.1-文件存储</li>
                    <li>5.1.1-TXT文本存储</li>
                    <li>5.1.2-JSON文件存储</li>
                    <li>5.1.3-CSV文件存储</li>
                    <li>5.2-关系型数据库存储</li>
                    <li>5.2.1-MySQL存储</li>
                    <li>5.3-非关系型数据库存储</li>
                    <li>5.3.1-MongoDB存储</li>
                    <li>5.3.2-Redis存储</li>
                    <li><strong>6-Ajax数据爬取</strong></li>
                    <li>6.1-什么是Ajax</li>
                    <li>6.2-Ajax分析方法</li>
                    <li>6.3-Ajax结果提取</li>
                    <li>6.4-分析Ajax爬取今日头条街拍美图</li>
                    <li><strong>7-动态渲染页面爬取</strong></li>
                    <li>7.1-Selenium的使用</li>
                    <li>7.2-Splash的使用</li>
                    <li>7.3-Splash负载均衡配置</li>
                    <li>7.4-使用Selenium爬取淘宝商品</li>
                    <li><strong>8-验证码的识别</strong></li>
                    <li>8.1-图形验证码的识别</li>
                    <li>8.2-极验滑动验证码的识别</li>
                    <li>8.3-点触验证码的识别</li>
                    <li>8.4-微博宫格验证码的识别</li>
                    <li><strong>9-代理的使用</strong></li>
                    <li>9.1-代理的设置</li>
                    <li>9.2-代理池的维护</li>
                    <li>9.3-付费代理的使用</li>
                    <li>9.4-ADSL拨号代理</li>
                    <li>9.5-使用代理爬取微信公众号文章</li>
                    <li><strong>10-模拟登录</strong></li>
                    <li>10.1-模拟登录并爬取GitHub</li>
                    <li>10.2-Cookies池的搭建</li>
                    <li><strong>11-App的爬取</strong></li>
                    <li>11.1-Charles的使用</li>
                    <li>11.2-mitmproxy的使用</li>
                    <li>11.3-mitmdump爬取“得到”App电子书信息</li>
                    <li>11.4-Appium的基本使用</li>
                    <li>11.5-Appium爬取微信朋友圈</li>
                    <li>11.6-Appium+mitmdump爬取京东商品</li>
                    <li><strong>12-pyspider框架的使用</strong></li>
                    <li>12.1-pyspider框架介绍</li>
                    <li>12.2-pyspider的基本使用</li>
                    <li>12.3-pyspider用法详解</li>
                    <li><strong>13-Scrapy框架的使用</strong></li>
                    <li>13.1-Scrapy框架介绍</li>
                    <li>13.2-Scrapy入门</li>
                    <li>13.3-Selector的用法</li>
                    <li>13.4-Spider的用法</li>
                    <li>13.5-Downloader Middleware的用法</li>
                    <li>13.6-Spider Middleware的用法</li>
                    <li>13.7-Item Pipeline的用法</li>
                    <li>13.8-Scrapy对接Selenium</li>
                    <li>13.9-Scrapy对接Splash</li>
                    <li>13.10-Scrapy通用爬虫</li>
                    <li>13.11-Scrapyrt的使用</li>
                    <li>13.12-Scrapy对接Docker</li>
                    <li>13.13-Scrapy爬取新浪微博</li>
                    <li><strong>14-分布式爬虫</strong></li>
                    <li>14.1-分布式爬虫原理</li>
                    <li>14.2-Scrapy-Redis源码解析</li>
                    <li>14.3-Scrapy分布式实现</li>
                    <li>14.4-Bloom Filter的对接</li>
                    <li><strong>15-分布式爬虫的部署</strong></li>
                    <li>15.1-Scrapyd分布式部署</li>
                    <li>15.2-Scrapyd-Client的使用</li>
                    <li>15.3-Scrapyd对接Docker</li>
                    <li>15.4-Scrapyd批量部署</li>
                    <li>15.5-Gerapy分布式管理</li>
                  </ul>
                  <h2 id="购买链接"><a href="#购买链接" class="headerlink" title="购买链接"></a><strong>购买链接</strong></h2>
                  <p>想必很多小伙伴已经等了很久了，之前预售那么久也一直迟迟没有货，发售就有不少网店又售空了，不过现在起不用担心了！</p>
                  <p>书籍现已在京东、天猫、当当等网店上架并全面供应啦，复制链接到浏览器打开或扫描二维码打开即可购买了！</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/京东二维码.png" alt=""></p>
                  <p> 京东商城</p>
                  <p><a href="https://item.jd.com/12333540.html" target="_blank" rel="noopener">https://item.jd.com/12333540.html</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/天猫二维码.png" alt=""></p>
                  <p> 天猫商城</p>
                  <p><a href="https://detail.tmall.com/item.htm?id=566699703917" target="_blank" rel="noopener">https://detail.tmall.com/item.htm?id=566699703917</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/当当二维码.png" alt=""></p>
                  <p>当当网</p>
                  <p><a href="http://product.dangdang.com/25249602.html" target="_blank" rel="noopener">http://product.dangdang.com/25249602.html</a></p>
                  <p>欢迎大家购买，谢谢支持！O(∩_∩)O</p>
                  <h2 id="免费预览"><a href="#免费预览" class="headerlink" title="免费预览"></a><strong>免费预览</strong></h2>
                  <p>不放心？想先看看有些啥，没问题！看这里： 免费章节试读： <a href="https://cuiqingcai.com/5052.html">https://cuiqingcai.com/5052.html</a> 将一直免费开放<strong>前7章节</strong>，欢迎大家试读！ 好了，接下来就是我们的福利环节啦~</p>
                  <h2 id="福利一：签名书！！！"><a href="#福利一：签名书！！！" class="headerlink" title="福利一：签名书！！！"></a><strong>福利一：签名书！！！</strong></h2>
                  <p>恭喜你看到这里了！那么接下来的福利时间就到了！后面还有两个福利不容错过~ 赠书活动第三波来袭，送 20 本作者亲笔签名书籍！！！ 活动流程（重要，请一定认真阅读）： <strong>公众号进击的Coder回复 “赠书” 获取序列码参与活动，2018.7.24 22:00 截止，逾期参与无效，请记住您的序列码，这是您的唯一标识。</strong> <strong>您可以转发活动页面邀请好友帮忙积攒人气值，最终取人气值前 20 位赠书，截止日期 2018.7.24 22:00，该时刻人气值前 20 位的朋友每人会获得签名书一本。</strong> 最终赠书名单会在微信公众号进击的Coder公布，届时请关注公众号消息！</p>
                  <h2 id="福利二：独家优惠！！！"><a href="#福利二：独家优惠！！！" class="headerlink" title="福利二：独家优惠！！！"></a><strong>福利二：独家优惠！！！</strong></h2>
                  <p>等等，你以为这就是全部福利吗？当然不是！除了抽奖送书，我们还拿到了拨号VPS知名品牌云立方的独家优惠，在公众号（进击的Coder ）中回复：“优惠券”，即可免费领取云立方50元主机优惠券，数量有限，先到先得！优惠券可在云立方官网（www.yunlifang.cn）购买动态IP拨号VPS时抵扣现金，有了它，爬虫代理易如反掌！ 你问我动态拨号VPS能做什么？应该怎么用在爬虫里？来这里了解一下： <a href="https://mp.weixin.qq.com/s?__biz=MzIzNzA4NDk3Nw==&amp;mid=2457735734&amp;idx=1&amp;sn=ca0d066d767570205daa9475e31cbc1f&amp;scene=21#wechat_redirect" target="_blank" rel="noopener">轻松获得海量稳定代理！ADSL拨号代理的搭建</a></p>
                  <h2 id="福利三：视频课程！！！"><a href="#福利三：视频课程！！！" class="headerlink" title="福利三：视频课程！！！"></a><strong>福利三：视频课程！！！</strong></h2>
                  <p>当然除了书籍，也有配套的视频课程，目前半价促销中，作者同样是崔庆才，二者结合学习效果更佳！限时优惠折扣中！扫描下图中二维码即可了解详情！ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/课程宣传图.png" alt=""> 最后也是最重要的就是参与活动的地址了！！！快来扫码回复领取属于你的福利吧！！！</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/gzh.jpg" alt=""></p>
                  <h2 id="特别致谢"><a href="#特别致谢" class="headerlink" title="特别致谢"></a>特别致谢</h2>
                  <p>最后特别感谢云立方、天善智能对本活动的大力支持！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-07-15 16:16:59" itemprop="dateCreated datePublished" datetime="2018-07-15T16:16:59+08:00">2018-07-15</time>
                </span>
                <span id="/6173.html" class="post-meta-item leancloud_visitors" data-flag-title="《Python3网络爬虫开发实战》第三波赠书活动来了！" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6160.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6160.html" class="post-title-link" itemprop="url">Python中异步协程的使用方法介绍</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="1-前言"><a href="#1-前言" class="headerlink" title="1. 前言"></a>1. 前言</h2>
                  <p>在执行一些 IO 密集型任务的时候，程序常常会因为等待 IO 而阻塞。比如在网络爬虫中，如果我们使用 requests 库来进行请求的话，如果网站响应速度过慢，程序一直在等待网站响应，最后导致其爬取效率是非常非常低的。 为了解决这类问题，本文就来探讨一下 Python 中异步协程来加速的方法，此种方法对于 IO 密集型任务非常有效。如将其应用到网络爬虫中，爬取效率甚至可以成百倍地提升。 注：本文协程使用 async/await 来实现，需要 Python 3.5 及以上版本。</p>
                  <h2 id="2-基本了解"><a href="#2-基本了解" class="headerlink" title="2. 基本了解"></a>2. 基本了解</h2>
                  <p>在了解异步协程之前，我们首先得了解一些基础概念，如阻塞和非阻塞、同步和异步、多进程和协程。</p>
                  <h3 id="2-1-阻塞"><a href="#2-1-阻塞" class="headerlink" title="2.1 阻塞"></a>2.1 阻塞</h3>
                  <p>阻塞状态指程序未得到所需计算资源时被挂起的状态。程序在等待某个操作完成期间，自身无法继续干别的事情，则称该程序在该操作上是阻塞的。 常见的阻塞形式有：网络 I/O 阻塞、磁盘 I/O 阻塞、用户输入阻塞等。阻塞是无处不在的，包括 CPU 切换上下文时，所有的进程都无法真正干事情，它们也会被阻塞。如果是多核 CPU 则正在执行上下文切换操作的核不可被利用。</p>
                  <h3 id="2-2-非阻塞"><a href="#2-2-非阻塞" class="headerlink" title="2.2 非阻塞"></a>2.2 非阻塞</h3>
                  <p>程序在等待某操作过程中，自身不被阻塞，可以继续运行干别的事情，则称该程序在该操作上是非阻塞的。 非阻塞并不是在任何程序级别、任何情况下都可以存在的。 仅当程序封装的级别可以囊括独立的子程序单元时，它才可能存在非阻塞状态。 非阻塞的存在是因为阻塞存在，正因为某个操作阻塞导致的耗时与效率低下，我们才要把它变成非阻塞的。</p>
                  <h3 id="2-3-同步"><a href="#2-3-同步" class="headerlink" title="2.3 同步"></a>2.3 同步</h3>
                  <p>不同程序单元为了完成某个任务，在执行过程中需靠某种通信方式以协调一致，称这些程序单元是同步执行的。 例如购物系统中更新商品库存，需要用“行锁”作为通信信号，让不同的更新请求强制排队顺序执行，那更新库存的操作是同步的。 简言之，同步意味着有序。</p>
                  <h3 id="2-4-异步"><a href="#2-4-异步" class="headerlink" title="2.4 异步"></a>2.4 异步</h3>
                  <p>为完成某个任务，不同程序单元之间过程中无需通信协调，也能完成任务的方式，不相关的程序单元之间可以是异步的。 例如，爬虫下载网页。调度程序调用下载程序后，即可调度其他任务，而无需与该下载任务保持通信以协调行为。不同网页的下载、保存等操作都是无关的，也无需相互通知协调。这些异步操作的完成时刻并不确定。 简言之，异步意味着无序。</p>
                  <h3 id="2-5-多进程"><a href="#2-5-多进程" class="headerlink" title="2.5 多进程"></a>2.5 多进程</h3>
                  <p>多进程就是利用 CPU 的多核优势，在同一时间并行地执行多个任务，可以大大提高执行效率。</p>
                  <h3 id="2-6-协程"><a href="#2-6-协程" class="headerlink" title="2.6 协程"></a>2.6 协程</h3>
                  <p>协程，英文叫做 Coroutine，又称微线程，纤程，协程是一种用户态的轻量级线程。 协程拥有自己的寄存器上下文和栈。协程调度切换时，将寄存器上下文和栈保存到其他地方，在切回来的时候，恢复先前保存的寄存器上下文和栈。因此协程能保留上一次调用时的状态，即所有局部状态的一个特定组合，每次过程重入时，就相当于进入上一次调用的状态。 协程本质上是个单进程，协程相对于多进程来说，无需线程上下文切换的开销，无需原子操作锁定及同步的开销，编程模型也非常简单。 我们可以使用协程来实现异步操作，比如在网络爬虫场景下，我们发出一个请求之后，需要等待一定的时间才能得到响应，但其实在这个等待过程中，程序可以干许多其他的事情，等到响应得到之后才切换回来继续处理，这样可以充分利用 CPU 和其他资源，这就是异步协程的优势。</p>
                  <h2 id="3-异步协程用法"><a href="#3-异步协程用法" class="headerlink" title="3. 异步协程用法"></a>3. 异步协程用法</h2>
                  <p>接下来让我们来了解下协程的实现，从 Python 3.4 开始，Python 中加入了协程的概念，但这个版本的协程还是以生成器对象为基础的，在 Python 3.5 则增加了 async/await，使得协程的实现更加方便。 Python 中使用协程最常用的库莫过于 asyncio，所以本文会以 asyncio 为基础来介绍协程的使用。 首先我们需要了解下面几个概念：</p>
                  <ul>
                    <li>event_loop：事件循环，相当于一个无限循环，我们可以把一些函数注册到这个事件循环上，当满足条件发生的时候，就会调用对应的处理方法。</li>
                    <li>coroutine：中文翻译叫协程，在 Python 中常指代为协程对象类型，我们可以将协程对象注册到时间循环中，它会被事件循环调用。我们可以使用 async 关键字来定义一个方法，这个方法在调用时不会立即被执行，而是返回一个协程对象。</li>
                    <li>task：任务，它是对协程对象的进一步封装，包含了任务的各个状态。</li>
                    <li>future：代表将来执行或没有执行的任务的结果，实际上和 task 没有本质区别。</li>
                  </ul>
                  <p>另外我们还需要了解 async/await 关键字，它是从 Python 3.5 才出现的，专门用于定义协程。其中，async 定义一个协程，await 用来挂起阻塞方法的执行。</p>
                  <h3 id="3-1-定义协程"><a href="#3-1-定义协程" class="headerlink" title="3.1 定义协程"></a>3.1 定义协程</h3>
                  <p>首先我们来定义一个协程，体验一下它和普通进程在实现上的不同之处，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">execute</span><span class="params">(x)</span>:</span></span><br><span class="line">    print(<span class="string">'Number:'</span>, x)</span><br><span class="line"></span><br><span class="line">coroutine = execute(<span class="number">1</span>)</span><br><span class="line">print(<span class="string">'Coroutine:'</span>, coroutine)</span><br><span class="line">print(<span class="string">'After calling execute'</span>)</span><br><span class="line"></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(coroutine)</span><br><span class="line">print(<span class="string">'After calling loop'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Coroutine: &lt;coroutine object<span class="built_in"> execute </span>at 0x1034cf830&gt;</span><br><span class="line">After calling execute</span><br><span class="line">Number: 1</span><br><span class="line">After calling loop</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先我们引入了 asyncio 这个包，这样我们才可以使用 async 和 await，然后我们使用 async 定义了一个 execute() 方法，方法接收一个数字参数，方法执行之后会打印这个数字。 随后我们直接调用了这个方法，然而这个方法并没有执行，而是返回了一个 coroutine 协程对象。随后我们使用 get_event_loop() 方法创建了一个事件循环 loop，并调用了 loop 对象的 run_until_complete() 方法将协程注册到事件循环 loop 中，然后启动。最后我们才看到了 execute() 方法打印了输出结果。 可见，async 定义的方法就会变成一个无法直接执行的 coroutine 对象，必须将其注册到事件循环中才可以执行。 上文我们还提到了 task，它是对 coroutine 对象的进一步封装，它里面相比 coroutine 对象多了运行状态，比如 running、finished 等，我们可以用这些状态来获取协程对象的执行情况。 在上面的例子中，当我们将 coroutine 对象传递给 run_until_complete() 方法的时候，实际上它进行了一个操作就是将 coroutine 封装成了 task 对象，我们也可以显式地进行声明，如下所示：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> execute(x):</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Number:'</span>, x)</span><br><span class="line">    <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line">coroutine = execute(<span class="number">1</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Coroutine:'</span>, coroutine)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'After calling execute'</span>)</span><br><span class="line"></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line"><span class="keyword">task</span> = loop.create_task(coroutine)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line">loop.run_until_complete(<span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'After calling loop'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Coroutine: &lt;coroutine object<span class="built_in"> execute </span>at 0x10e0f7830&gt;</span><br><span class="line">After calling execute</span><br><span class="line">Task: &lt;Task pending coro=&lt;execute() running at demo.py:4&gt;&gt;</span><br><span class="line">Number: 1</span><br><span class="line">Task: &lt;Task finished coro=&lt;execute() done, defined at demo.py:4&gt; result=1&gt;</span><br><span class="line">After calling loop</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了 loop 对象之后，接着调用了它的 create_task() 方法将 coroutine 对象转化为了 task 对象，随后我们打印输出一下，发现它是 pending 状态。接着我们将 task 对象添加到事件循环中得到执行，随后我们再打印输出一下 task 对象，发现它的状态就变成了 finished，同时还可以看到其 result 变成了 1，也就是我们定义的 execute() 方法的返回结果。 另外定义 task 对象还有一种方式，就是直接通过 asyncio 的 ensure_future() 方法，返回结果也是 task 对象，这样的话我们就可以不借助于 loop 来定义，即使我们还没有声明 loop 也可以提前定义好 task 对象，写法如下：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> execute(x):</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Number:'</span>, x)</span><br><span class="line">    <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line">coroutine = execute(<span class="number">1</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Coroutine:'</span>, coroutine)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'After calling execute'</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">task</span> = asyncio.ensure_future(coroutine)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(<span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'After calling loop'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Coroutine: &lt;coroutine object<span class="built_in"> execute </span>at 0x10aa33830&gt;</span><br><span class="line">After calling execute</span><br><span class="line">Task: &lt;Task pending coro=&lt;execute() running at demo.py:4&gt;&gt;</span><br><span class="line">Number: 1</span><br><span class="line">Task: &lt;Task finished coro=&lt;execute() done, defined at demo.py:4&gt; result=1&gt;</span><br><span class="line">After calling loop</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>发现其效果都是一样的。</p>
                  <h3 id="3-2-绑定回调"><a href="#3-2-绑定回调" class="headerlink" title="3.2 绑定回调"></a>3.2 绑定回调</h3>
                  <p>另外我们也可以为某个 task 绑定一个回调方法，来看下面的例子：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> request():</span><br><span class="line">    url = <span class="string">'https://www.baidu.com'</span></span><br><span class="line">    status = requests.get(url)</span><br><span class="line">    <span class="keyword">return</span> status</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> callback(<span class="keyword">task</span>):</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Status:'</span>, <span class="keyword">task</span>.result())</span><br><span class="line"></span><br><span class="line">coroutine = request()</span><br><span class="line"><span class="keyword">task</span> = asyncio.ensure_future(coroutine)</span><br><span class="line"><span class="keyword">task</span>.add_done_callback(callback)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line"></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(<span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们定义了一个 request() 方法，请求了百度，返回状态码，但是这个方法里面我们没有任何 print() 语句。随后我们定义了一个 callback() 方法，这个方法接收一个参数，是 task 对象，然后调用 print() 方法打印了 task 对象的结果。这样我们就定义好了一个 coroutine 对象和一个回调方法，我们现在希望的效果是，当 coroutine 对象执行完毕之后，就去执行声明的 callback() 方法。 那么它们二者怎样关联起来呢？很简单，只需要调用 add_done_callback() 方法即可，我们将 callback() 方法传递给了封装好的 task 对象，这样当 task 执行完毕之后就可以调用 callback() 方法了，同时 task 对象还会作为参数传递给 callback() 方法，调用 task 对象的 result() 方法就可以获取返回结果了。 运行结果：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Task</span>: &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt; cb=[callback() <span class="keyword">at</span> demo.py:<span class="number">11</span>]&gt;</span><br><span class="line">Status: &lt;Response [<span class="number">200</span>]&gt;</span><br><span class="line"><span class="keyword">Task</span>: &lt;<span class="keyword">Task</span> finished coro=&lt;request() done, defined <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt; result=&lt;Response [<span class="number">200</span>]&gt;&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实际上不用回调方法，直接在 task 运行完毕之后也可以直接调用 result() 方法获取结果，如下所示：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> request():</span><br><span class="line">    url = <span class="string">'https://www.baidu.com'</span></span><br><span class="line">    status = requests.get(url)</span><br><span class="line">    <span class="keyword">return</span> status</span><br><span class="line"></span><br><span class="line">coroutine = request()</span><br><span class="line"><span class="keyword">task</span> = asyncio.ensure_future(coroutine)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line"></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(<span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task:'</span>, <span class="keyword">task</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Task Result:'</span>, <span class="keyword">task</span>.result())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果是一样的：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Task</span>: &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">4</span>&gt;&gt;</span><br><span class="line"><span class="keyword">Task</span>: &lt;<span class="keyword">Task</span> finished coro=&lt;request() done, defined <span class="keyword">at</span> demo.py:<span class="number">4</span>&gt; result=&lt;Response [<span class="number">200</span>]&gt;&gt;</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="3-3-多任务协程"><a href="#3-3-多任务协程" class="headerlink" title="3.3 多任务协程"></a>3.3 多任务协程</h3>
                  <p>上面的例子我们只执行了一次请求，如果我们想执行多次请求应该怎么办呢？我们可以定义一个 task 列表，然后使用 asyncio 的 wait() 方法即可执行，看下面的例子：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">request</span><span class="params">()</span>:</span></span><br><span class="line">    url = <span class="string">'https://www.baidu.com'</span></span><br><span class="line">    status = requests.get(url)</span><br><span class="line">    <span class="keyword">return</span> status</span><br><span class="line"></span><br><span class="line">tasks = [asyncio.ensure_future(request()) <span class="keyword">for</span> _ <span class="keyword">in</span> range(<span class="number">5</span>)]</span><br><span class="line">print(<span class="string">'Tasks:'</span>, tasks)</span><br><span class="line"></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> task <span class="keyword">in</span> tasks:</span><br><span class="line">    print(<span class="string">'Task Result:'</span>, task.result())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用一个 for 循环创建了五个 task，组成了一个列表，然后把这个列表首先传递给了 asyncio 的 wait() 方法，然后再将其注册到时间循环中，就可以发起五个任务了。最后我们再将任务的运行结果输出出来，运行结果如下：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tasks: [&lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt;&gt;, &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt;&gt;, &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt;&gt;, &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt;&gt;, &lt;<span class="keyword">Task</span> pending coro=&lt;request() running <span class="keyword">at</span> demo.py:<span class="number">5</span>&gt;&gt;]</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br><span class="line"><span class="keyword">Task</span> Result: &lt;Response [<span class="number">200</span>]&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到五个任务被顺次执行了，并得到了运行结果。</p>
                  <h3 id="3-4-协程实现"><a href="#3-4-协程实现" class="headerlink" title="3.4 协程实现"></a>3.4 协程实现</h3>
                  <p>前面说了这么一通，又是 async，又是 coroutine，又是 task，又是 callback，但似乎并没有看出协程的优势啊？反而写法上更加奇怪和麻烦了，别急，上面的案例只是为后面的使用作铺垫，接下来我们正式来看下协程在解决 IO 密集型任务上有怎样的优势吧！ 上面的代码中，我们用一个网络请求作为示例，这就是一个耗时等待的操作，因为我们请求网页之后需要等待页面响应并返回结果。耗时等待的操作一般都是 IO 操作，比如文件读取、网络请求等等。协程对于处理这种操作是有很大优势的，当遇到需要等待的情况的时候，程序可以暂时挂起，转而去执行其他的操作，从而避免一直等待一个程序而耗费过多的时间，充分利用资源。 为了表现出协程的优势，我们需要先创建一个合适的实验环境，最好的方法就是模拟一个需要等待一定时间才可以获取返回结果的网页，上面的代码中使用了百度，但百度的响应太快了，而且响应速度也会受本机网速影响，所以最好的方式是自己在本地模拟一个慢速服务器，这里我们选用 Flask。 如果没有安装 Flask 的话可以执行如下命令安装：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> flask</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后编写服务器代码如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> flask <span class="keyword">import</span> Flask</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">app = Flask(__name__)</span><br><span class="line"></span><br><span class="line">@app.route(<span class="string">'/'</span>)</span><br><span class="line">def <span class="keyword">index</span>():</span><br><span class="line">    <span class="type">time</span>.sleep(<span class="number">3</span>)</span><br><span class="line">    <span class="keyword">return</span> <span class="string">'Hello!'</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    app.run(threaded=<span class="keyword">True</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了一个 Flask 服务，主入口是 index() 方法，方法里面先调用了 sleep() 方法休眠 3 秒，然后接着再返回结果，也就是说，每次请求这个接口至少要耗时 3 秒，这样我们就模拟了一个慢速的服务接口。 注意这里服务启动的时候，run() 方法加了一个参数 threaded，这表明 Flask 启动了多线程模式，不然默认是只有一个线程的。如果不开启多线程模式，同一时刻遇到多个请求的时候，只能顺次处理，这样即使我们使用协程异步请求了这个服务，也只能一个一个排队等待，瓶颈就会出现在服务端。所以，多线程模式是有必要打开的。 启动之后，Flask 应该默认会在 127.0.0.1:5000 上运行，运行之后控制台输出结果如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"> * Running <span class="keyword">on</span> http://<span class="number">127.0</span>.<span class="number">0.1</span>:<span class="number">5000</span>/ (Press CTRL+C <span class="keyword">to</span> <span class="keyword">quit</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们再重新使用上面的方法请求一遍：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">start = <span class="type">time</span>.time()</span><br><span class="line"></span><br><span class="line">async def request():</span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    print(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    response = requests.<span class="keyword">get</span>(url)</span><br><span class="line">    print(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, response.text)</span><br><span class="line"></span><br><span class="line">tasks = [asyncio.ensure_future(request()) <span class="keyword">for</span> _ <span class="keyword">in</span> range(<span class="number">5</span>)]</span><br><span class="line"><span class="keyword">loop</span> = asyncio.get_event_loop()</span><br><span class="line"><span class="keyword">loop</span>.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line">end = <span class="type">time</span>.time()</span><br><span class="line">print(<span class="string">'Cost time:'</span>, <span class="keyword">end</span> - <span class="keyword">start</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们还是创建了五个 task，然后将 task 列表传给 wait() 方法并注册到时间循环中执行。 运行结果如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Cost <span class="string">time:</span> <span class="number">15.049368143081665</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现和正常的请求并没有什么两样，依然还是顺次执行的，耗时 15 秒，平均一个请求耗时 3 秒，说好的异步处理呢？ 其实，要实现异步处理，我们得先要有挂起的操作，当一个任务需要等待 IO 结果的时候，可以挂起当前任务，转而去执行其他任务，这样我们才能充分利用好资源，上面方法都是一本正经的串行走下来，连个挂起都没有，怎么可能实现异步？想太多了。 要实现异步，接下来我们再了解一下 await 的用法，使用 await 可以将耗时等待的操作挂起，让出控制权。当协程执行的时候遇到 await，时间循环就会将本协程挂起，转而去执行别的协程，直到其他的协程挂起或执行完毕。 所以，我们可能会将代码中的 request() 方法改成如下的样子：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">request</span><span class="params">()</span>:</span></span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    print(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    response = <span class="keyword">await</span> requests.get(url)</span><br><span class="line">    print(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, response.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>仅仅是在 requests 前面加了一个 await，然而执行以下代码，会得到如下报错：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Cost <span class="string">time:</span> <span class="number">15.048935890197754</span></span><br><span class="line">Task exception was never retrieved</span><br><span class="line"><span class="string">future:</span> &lt;Task finished coro=&lt;request() done, defined at demo.<span class="string">py:</span><span class="number">7</span>&gt; exception=TypeError(<span class="string">"object Response can't be used in 'await' expression"</span>,)&gt;</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File <span class="string">"demo.py"</span>, line <span class="number">10</span>, <span class="keyword">in</span> request</span><br><span class="line">    status = await requests.get(url)</span><br><span class="line"><span class="string">TypeError:</span> object Response can<span class="string">'t be used in '</span>await<span class="string">' expression</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这次它遇到 await 方法确实挂起了，也等待了，但是最后却报了这么个错，这个错误的意思是 requests 返回的 Response 对象不能和 await 一起使用，为什么呢？因为根据官方文档说明，await 后面的对象必须是如下格式之一：</p>
                  <ul>
                    <li>A native coroutine object returned from a native coroutine function，一个原生 coroutine 对象。</li>
                    <li>A generator-based coroutine object returned from a function decorated with types.coroutine()，一个由 types.coroutine() 修饰的生成器，这个生成器可以返回 coroutine 对象。</li>
                    <li>An object with an await<strong> method returning an iterator，一个包含 </strong>await 方法的对象返回的一个迭代器。</li>
                  </ul>
                  <p>可以参见：<a href="https://www.python.org/dev/peps/pep-0492/#await-expression" target="_blank" rel="noopener">https://www.python.org/dev/peps/pep-0492/#await-expression</a>。 reqeusts 返回的 Response 不符合上面任一条件，因此就会报上面的错误了。 那么有的小伙伴就发现了，既然 await 后面可以跟一个 coroutine 对象，那么我用 async 把请求的方法改成 coroutine 对象不就可以了吗？所以就改写成如下的样子：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">start = <span class="type">time</span>.time()</span><br><span class="line"></span><br><span class="line">async def <span class="keyword">get</span>(url):</span><br><span class="line">    <span class="keyword">return</span> requests.<span class="keyword">get</span>(url)</span><br><span class="line"></span><br><span class="line">async def request():</span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    print(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    response = await <span class="keyword">get</span>(url)</span><br><span class="line">    print(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, response.text)</span><br><span class="line"></span><br><span class="line">tasks = [asyncio.ensure_future(request()) <span class="keyword">for</span> _ <span class="keyword">in</span> range(<span class="number">5</span>)]</span><br><span class="line"><span class="keyword">loop</span> = asyncio.get_event_loop()</span><br><span class="line"><span class="keyword">loop</span>.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line">end = <span class="type">time</span>.time()</span><br><span class="line">print(<span class="string">'Cost time:'</span>, <span class="keyword">end</span> - <span class="keyword">start</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们将请求页面的方法独立出来，并用 async 修饰，这样就得到了一个 coroutine 对象，我们运行一下看看：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Cost <span class="string">time:</span> <span class="number">15.134317874908447</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>还是不行，它还不是异步执行，也就是说我们仅仅将涉及 IO 操作的代码封装到 async 修饰的方法里面是不可行的！我们必须要使用支持异步操作的请求方式才可以实现真正的异步，所以这里就需要 aiohttp 派上用场了。</p>
                  <h3 id="3-5-使用-aiohttp"><a href="#3-5-使用-aiohttp" class="headerlink" title="3.5 使用 aiohttp"></a>3.5 使用 aiohttp</h3>
                  <p>aiohttp 是一个支持异步请求的库，利用它和 asyncio 配合我们可以非常方便地实现异步请求操作。 安装方式如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> aiohttp</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>官方文档链接为：<a href="https://aiohttp.readthedocs.io/" target="_blank" rel="noopener">https://aiohttp.readthedocs.io/</a>，它分为两部分，一部分是 Client，一部分是 Server，详细的内容可以参考官方文档。 下面我们将 aiohttp 用上来，将代码改成如下样子：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">start = <span class="type">time</span>.time()</span><br><span class="line"></span><br><span class="line">async def <span class="keyword">get</span>(url):</span><br><span class="line">    <span class="keyword">session</span> = aiohttp.ClientSession()</span><br><span class="line">    response = await <span class="keyword">session</span>.<span class="keyword">get</span>(url)</span><br><span class="line">    result = await response.text()</span><br><span class="line">    <span class="keyword">session</span>.<span class="keyword">close</span>()</span><br><span class="line">    <span class="keyword">return</span> result</span><br><span class="line"></span><br><span class="line">async def request():</span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    print(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    result = await <span class="keyword">get</span>(url)</span><br><span class="line">    print(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, result)</span><br><span class="line"></span><br><span class="line">tasks = [asyncio.ensure_future(request()) <span class="keyword">for</span> _ <span class="keyword">in</span> range(<span class="number">5</span>)]</span><br><span class="line"><span class="keyword">loop</span> = asyncio.get_event_loop()</span><br><span class="line"><span class="keyword">loop</span>.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line">end = <span class="type">time</span>.time()</span><br><span class="line">print(<span class="string">'Cost time:'</span>, <span class="keyword">end</span> - <span class="keyword">start</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们将请求库由 requests 改成了 aiohttp，通过 aiohttp 的 ClientSession 类的 get() 方法进行请求，结果如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Waiting <span class="keyword">for</span> <span class="string">http:</span><span class="comment">//127.0.0.1:5000</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Get response from <span class="string">http:</span><span class="comment">//127.0.0.1:5000 Result: Hello!</span></span><br><span class="line">Cost <span class="string">time:</span> <span class="number">3.0199508666992188</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>成功了！我们发现这次请求的耗时由 15 秒变成了 3 秒，耗时直接变成了原来的 1/5。 代码里面我们使用了 await，后面跟了 get() 方法，在执行这五个协程的时候，如果遇到了 await，那么就会将当前协程挂起，转而去执行其他的协程，直到其他的协程也挂起或执行完毕，再进行下一个协程的执行。 开始运行时，时间循环会运行第一个 task，针对第一个 task 来说，当执行到第一个 await 跟着的 get() 方法时，它被挂起，但这个 get() 方法第一步的执行是非阻塞的，挂起之后立马被唤醒，所以立即又进入执行，创建了 ClientSession 对象，接着遇到了第二个 await，调用了 session.get() 请求方法，然后就被挂起了，由于请求需要耗时很久，所以一直没有被唤醒，好第一个 task 被挂起了，那接下来该怎么办呢？事件循环会寻找当前未被挂起的协程继续执行，于是就转而执行第二个 task 了，也是一样的流程操作，直到执行了第五个 task 的 session.get() 方法之后，全部的 task 都被挂起了。所有 task 都已经处于挂起状态，那咋办？只好等待了。3 秒之后，几个请求几乎同时都有了响应，然后几个 task 也被唤醒接着执行，输出请求结果，最后耗时，3 秒！ 怎么样？这就是异步操作的便捷之处，当遇到阻塞式操作时，任务被挂起，程序接着去执行其他的任务，而不是傻傻地等着，这样可以充分利用 CPU 时间，而不必把时间浪费在等待 IO 上。 有人就会说了，既然这样的话，在上面的例子中，在发出网络请求后，既然接下来的 3 秒都是在等待的，在 3 秒之内，CPU 可以处理的 task 数量远不止这些，那么岂不是我们放 10 个、20 个、50 个、100 个、1000 个 task 一起执行，最后得到所有结果的耗时不都是 3 秒左右吗？因为这几个任务被挂起后都是一起等待的。 理论来说确实是这样的，不过有个前提，那就是服务器在同一时刻接受无限次请求都能保证正常返回结果，也就是服务器无限抗压，另外还要忽略 IO 传输时延，确实可以做到无限 task 一起执行且在预想时间内得到结果。 我们这里将 task 数量设置成 100，再试一下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">tasks</span> = [asyncio.ensure_future(request()) for _ in range(<span class="number">100</span>)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>耗时结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Cost</span> <span class="type">time</span>: <span class="number">3.106252670288086</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后运行时间也是在 3 秒左右，当然多出来的时间就是 IO 时延了。 可见，使用了异步协程之后，我们几乎可以在相同的时间内实现成百上千倍次的网络请求，把这个运用在爬虫中，速度提升可谓是非常可观了。</p>
                  <h3 id="3-6-与单进程、多进程对比"><a href="#3-6-与单进程、多进程对比" class="headerlink" title="3.6 与单进程、多进程对比"></a>3.6 与单进程、多进程对比</h3>
                  <p>可能有的小伙伴非常想知道上面的例子中，如果 100 次请求，不是用异步协程的话，使用单进程和多进程会耗费多少时间，我们来测试一下： 首先来测试一下单进程的时间：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">start = <span class="type">time</span>.time()</span><br><span class="line"></span><br><span class="line">def request():</span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    print(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    result = requests.<span class="keyword">get</span>(url).text</span><br><span class="line">    print(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, result)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> range(<span class="number">100</span>):</span><br><span class="line">    request()</span><br><span class="line"></span><br><span class="line">end = <span class="type">time</span>.time()</span><br><span class="line">print(<span class="string">'Cost time:'</span>, <span class="keyword">end</span> - <span class="keyword">start</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后耗时：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Cost</span> <span class="type">time</span>: <span class="number">305.16639709472656</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们使用多进程来测试下，使用 multiprocessing 库：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line">import time</span><br><span class="line">import multiprocessing</span><br><span class="line"></span><br><span class="line">start = time.time()</span><br><span class="line"></span><br><span class="line">def request(_):</span><br><span class="line">    url = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    <span class="builtin-name">print</span>(<span class="string">'Waiting for'</span>, url)</span><br><span class="line">    result = requests.<span class="builtin-name">get</span>(url).text</span><br><span class="line">    <span class="builtin-name">print</span>(<span class="string">'Get response from'</span>, url, <span class="string">'Result:'</span>, result)</span><br><span class="line"></span><br><span class="line">cpu_count = multiprocessing.cpu_count()</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Cpu count:'</span>, cpu_count)</span><br><span class="line">pool = multiprocessing.Pool(cpu_count)</span><br><span class="line">pool.map(request, range(100))</span><br><span class="line"></span><br><span class="line">end = time.time()</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Cost time:'</span>, end - start)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我使用了multiprocessing 里面的 Pool 类，即进程池。我的电脑的 CPU 个数是 8 个，这里的进程池的大小就是 8。 运行时间：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Cost</span> <span class="type">time</span>: <span class="number">48.17306900024414</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见 multiprocessing 相比单线程来说，还是可以大大提高效率的。</p>
                  <h3 id="3-7-与多进程的结合"><a href="#3-7-与多进程的结合" class="headerlink" title="3.7 与多进程的结合"></a>3.7 与多进程的结合</h3>
                  <p>既然异步协程和多进程对网络请求都有提升，那么为什么不把二者结合起来呢？在最新的 PyCon 2018 上，来自 Facebook 的 John Reese 介绍了 asyncio 和 multiprocessing 各自的特点，并开发了一个新的库，叫做 aiomultiprocess，感兴趣的可以了解下：<a href="https://www.youtube.com/watch?v=0kXaLh8Fz3k" target="_blank" rel="noopener">https://www.youtube.com/watch?v=0kXaLh8Fz3k</a>。 这个库的安装方式是：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> aiomultiprocess</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>需要 Python 3.6 及更高版本才可使用。 使用这个库，我们可以将上面的例子改写如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import asyncio</span><br><span class="line">import aiohttp</span><br><span class="line">import time</span><br><span class="line">from aiomultiprocess import Pool</span><br><span class="line"></span><br><span class="line"><span class="keyword">start</span> = time.time()</span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> <span class="keyword">get</span>(<span class="keyword">url</span>):</span><br><span class="line">    <span class="keyword">session</span> = aiohttp.ClientSession()</span><br><span class="line">    response = await session.get(<span class="keyword">url</span>)</span><br><span class="line">    <span class="keyword">result</span> = await response.text()</span><br><span class="line">    session.close()</span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">result</span></span><br><span class="line"></span><br><span class="line">async <span class="keyword">def</span> request():</span><br><span class="line">    <span class="keyword">url</span> = <span class="string">'http://127.0.0.1:5000'</span></span><br><span class="line">    urls = [<span class="keyword">url</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="keyword">range</span>(<span class="number">100</span>)]</span><br><span class="line">    async <span class="keyword">with</span> Pool() <span class="keyword">as</span> pool:</span><br><span class="line">        <span class="keyword">result</span> = await pool.map(<span class="keyword">get</span>, urls)</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">result</span></span><br><span class="line"></span><br><span class="line">coroutine = request()</span><br><span class="line">task = asyncio.ensure_future(coroutine)</span><br><span class="line"><span class="keyword">loop</span> = asyncio.get_event_loop()</span><br><span class="line">loop.run_until_complete(task)</span><br><span class="line"></span><br><span class="line"><span class="keyword">end</span> = time.time()</span><br><span class="line">print(<span class="string">'Cost time:'</span>, <span class="keyword">end</span> - <span class="keyword">start</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就会同时使用多进程和异步协程进行请求，当然最后的结果其实和异步是差不多的：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Cost</span> <span class="type">time</span>: <span class="number">3.1156570434570312</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因为我的测试接口的原因，最快的响应也是 3 秒，所以这部分多余的时间基本都是 IO 传输时延。但在真实情况下，我们在做爬取的时候遇到的情况千变万化，一方面我们使用异步协程来防止阻塞，另一方面我们使用 multiprocessing 来利用多核成倍加速，节省时间其实还是非常可观的。 以上便是 Python 中协程的基本用法，希望对大家有帮助。</p>
                  <h2 id="4-参考来源"><a href="#4-参考来源" class="headerlink" title="4. 参考来源"></a>4. 参考来源</h2>
                  <ul>
                    <li><a href="http://python.jobbole.com/87310/" target="_blank" rel="noopener">http://python.jobbole.com/87310/</a></li>
                    <li><a href="https://www.cnblogs.com/xybaby/p/6406191.html" target="_blank" rel="noopener">https://www.cnblogs.com/xybaby/p/6406191.html</a></li>
                    <li><a href="http://python.jobbole.com/88291/" target="_blank" rel="noopener">http://python.jobbole.com/88291/</a></li>
                    <li><a href="http://lotabout.me/2017/understand-python-asyncio/" target="_blank" rel="noopener">http://lotabout.me/2017/understand-python-asyncio/</a></li>
                    <li><a href="https://segmentfault.com/a/1190000008814676" target="_blank" rel="noopener">https://segmentfault.com/a/1190000008814676</a></li>
                    <li><a href="https://www.cnblogs.com/animalize/p/4738941.html" target="_blank" rel="noopener">https://www.cnblogs.com/animalize/p/4738941.html</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-07-06 01:38:16" itemprop="dateCreated datePublished" datetime="2018-07-06T01:38:16+08:00">2018-07-06</time>
                </span>
                <span id="/6160.html" class="post-meta-item leancloud_visitors" data-flag-title="Python中异步协程的使用方法介绍" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>16k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>14 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6146.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/6146.html" class="post-title-link" itemprop="url">推荐一些Mac上比较好用的软件</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>最近有一个朋友刚入手了 Mac，准备专门搞开发用，让我给他推荐几款软件，然后我就把我的 Launchpad 截图发给了他，他看到这密密麻麻的软件完全不知所措。<a href="https://github.com/Germey/AI/blob/master/assets/2018-07-03-01-51-59.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-07-03-01-51-59.jpg" alt=""></a> 于是乎，我就大略整理了一些我比较推荐的几款软件，同时分享给大家，希望对大家有所帮助！ 下面的一些软件都是我个人比较喜欢的，其实还有很多其他的恕不能一一列举了，如果大家有其他推荐的欢迎留言给我，谢谢！</p>
                  <h2 id="日常工具"><a href="#日常工具" class="headerlink" title="日常工具"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E6%97%A5%E5%B8%B8%E5%B7%A5%E5%85%B7" target="_blank" rel="noopener"></a>日常工具</h2>
                  <p>一些日常工具在这里我就不一一列举了，大部分使用 Mac 的小伙伴都会安装，比如 QQ、微信、Chrome 浏览器、网易云音乐、迅雷等等，这些在 Windows 上也几乎都是必备软件，这里就不再展开说明了。</p>
                  <h2 id="效率工具"><a href="#效率工具" class="headerlink" title="效率工具"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E6%95%88%E7%8E%87%E5%B7%A5%E5%85%B7" target="_blank" rel="noopener"></a>效率工具</h2>
                  <p>效率工具顾名思义，可以方便和简化 Mac 的操作，提高生产工作效率的工具，下面推荐几款我比较常用的。</p>
                  <h3 id="Alfred"><a href="#Alfred" class="headerlink" title="Alfred"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#alfred" target="_blank" rel="noopener"></a>Alfred</h3>
                  <p>首推 Alfred，可以说是 Mac 必备软件，利用它我们可以快速地进行各种操作，大幅提高工作效率，如快速打开某个软件、快速打开某个链接、快速搜索某个文档，快速定位某个文件，快速查看本机 IP，快速定义某个色值，几乎你能想到的都能对接实现。 这些快速功能是怎么实现的呢？实际上是 Alfred 对接了很多 Workflow，我们可以使用 Workflow 方便地进行功能扩展，一些比较优秀的 Workflow 已经有人专门做过整理了，可以参见：<a href="https://github.com/zenorocha/alfred-workflows" target="_blank" rel="noopener">https://github.com/zenorocha/alfred-workflows</a>。 推荐指数：★★★★★</p>
                  <h3 id="Todoist"><a href="#Todoist" class="headerlink" title="Todoist"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#todoist" target="_blank" rel="noopener"></a>Todoist</h3>
                  <p>大家肯定也在使用各种 Todo List 的软件，这种软件其实也是五花八门，经过我本人试用，我觉得 Todoist 这款软件是最方便的。 它支持各种类型的任务定制，还可以设置分组、优先级、Deadline、执行人员、提醒、协作、效率统计等功能。另外它的各个平台支持真是异常地全啊，网页、PC、移动端就不用说了，都必须有的，另外它还有浏览器插件版、电邮版、可穿戴设备（如 Apple Watch、Google Wear）版，另外他还可以和 Mac 的日历事件进行同步，日历添加的事件也会自动添加到 Todoist 里面，非常方便，是目前我体验过的最好用的一款。 这款软件个人推荐购买专业版解锁全部功能，一个月 3 刀，但个人觉得确实非常值。 推荐指数：★★★★☆</p>
                  <h3 id="Paste"><a href="#Paste" class="headerlink" title="Paste"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#paste" target="_blank" rel="noopener"></a>Paste</h3>
                  <p>Mac 上默认只有一个粘贴板，当我们新复制了一段文字之后，如果我们想再找寻之前复制的历史记录就找不到了，这其实是很反人类的。 好在 Paste 这款软件帮我们解决了这个问题，它可以保存我们粘贴板的历史记录，等需要粘贴某个内容的时候只需要呼出 Paste 历史粘贴板，然后选择某个特定的内容粘贴就好了，另外它还支持文本格式调整粘贴板分类和搜索，还可以支持快速便捷粘贴。有了它，妈妈再也不用担心我的粘贴板丢失了！ 推荐指数：★★★★★</p>
                  <h3 id="Synergy"><a href="#Synergy" class="headerlink" title="Synergy"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#synergy" target="_blank" rel="noopener"></a>Synergy</h3>
                  <p>工作时我会使用公司的台式机，是 Windows 系统，另外自己的个人笔记本 Mac 也会放在旁边，两台 PC 有时候会交替使用，但是我总不能配两套键盘和鼠标吧，这样就显得累赘了，而且也没那么多地方放啊。 有了 Synergy，我们可以将两台 PC 关联，实现键盘鼠标共享。我们可以使用一套键盘和鼠标来操作两台 PC，注意这是两个完全独立的 PC，各自有各自的屏幕和系统，使用 Synergy 我们可以做到一套键鼠同时控制两台电脑，鼠标可以直接从一台电脑的屏幕滑动到另一台电脑屏幕上，同时键盘、粘贴板也都是共享的。 设想这么个情景，我在我的台式机 Windows 上打开了一个页面，需要让我输入一个很长的序列号，而这个序列号又恰巧存在 Mac 上，这时如果有了 Synergy 将二者关联，我们只需要把鼠标从 Windows 的屏幕上直接滑动到 Mac 的屏幕上，选中序列号，然后键盘按下复制的快捷键，然后再把鼠标移回 Windows，粘贴即可，一气呵成。而不必再想办法发消息传输了，大大提高效率。 推荐指数：★★★★</p>
                  <h3 id="Feedly、Reeder"><a href="#Feedly、Reeder" class="headerlink" title="Feedly、Reeder"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#feedlyreeder" target="_blank" rel="noopener"></a>Feedly、Reeder</h3>
                  <p>博客现在已经越来越多了，越来越多的人开始在博客上发表文章，而当我们遇到优质的博客时，我们还想随时知道博客的发表动态，一旦有新文章发表我们想立马得到相关动态，这样可以实现吗？ 肯定是可行的，现在绝大多数博客都有 RSS 订阅功能，有了它我们可以订阅自己喜欢的博客，这里我使用的 RSS 订阅工具就是 Feedly，利用它我可以很轻松地添加自己喜欢的博客或论坛到自己的 Feed 流里面，一旦有文章更新，我就会收到相应提示。 但是 Feedly 有个小问题，就是在国内速度太慢了，所以我又使用 Reeder 将 Feedly 里面的 Feed 流做了转接，它可以添加 Feedly 源，并带有灵活的分类、标记等管理功能，还支持各种预览方式，还支持存储到 Pocket，还有各种分享方式，功能十分齐全。 总之，推荐 Feedly 来添加自己喜欢的博客，用 Reeder 来阅读订阅的内容，双剑合璧，另外 Reeder 对移动版的支持也很不错，可以体验一下。 推荐指数：★★★★</p>
                  <h3 id="Mindnode"><a href="#Mindnode" class="headerlink" title="Mindnode"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#mindnode" target="_blank" rel="noopener"></a>Mindnode</h3>
                  <p>有时候在思考问题的时候我们想要把一些思路记录下来，另外在做一些概要设计的时候需要把概要图大体描述出来，这时候画一个思维导图再合适不过了，比如你现在读的这篇文章就很适合用一个思维导图画一下。 画思维导图我个人比较喜欢的一款软件是 Mindnode，觉得比较简洁好用，当然也有不少人使用 XMind，也很不错。可能是先入为主，也可能是界面设计风格，我个人更加偏向于使用 Mindnode。 推荐指数：★★★★</p>
                  <h3 id="1Password"><a href="#1Password" class="headerlink" title="1Password"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#1password" target="_blank" rel="noopener"></a>1Password</h3>
                  <p>随着年龄的增长，我们可能变得越来越忘事了。另外还有些反人类的网站密码必须要至少大写、小写、数字、特殊符号，有的还要求不少于多少位，有的还要求我么能定时更换密码，还不能与之前用过的相同！这会使得我们之前预想设计的很多密码都没法用了。另外网站又这么多，谁又能把网站的密码都记下来啊？ 这时候我们就需要一款专门管理密码的软件，我个人推荐一款叫做 1Password，有了它我们可以将各个平台的密码保存起来，同时它还可以根据我们的要求帮我们随机生成一些密码并保存，这对注册一些新网站非常有用，同时使用随机的密码还降低了撞库的风险，不然一个平台的密码被盗了，其他平台用的同样的密码的话，就很不安全了。 1Password 还支持各种平台，如网页、PC、移动版都通通完美支持，实现密码云同步，妈妈再也不用担心我忘记密码了！ 推荐指数：★★★★</p>
                  <h2 id="系统工具"><a href="#系统工具" class="headerlink" title="系统工具"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E7%B3%BB%E7%BB%9F%E5%B7%A5%E5%85%B7" target="_blank" rel="noopener"></a>系统工具</h2>
                  <p>下面介绍的两款系统工具软件几乎是装机必备的。</p>
                  <h3 id="Tuxera-NTFS-For-Mac"><a href="#Tuxera-NTFS-For-Mac" class="headerlink" title="Tuxera NTFS For Mac"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#tuxera-ntfs-for-mac" target="_blank" rel="noopener"></a>Tuxera NTFS For Mac</h3>
                  <p>用了 Mac，我们在使用移动硬盘的时候可能会遇到一个无法传输数据（如拷贝文件）的问题，这是因为部分移动硬盘是 NTFS 格式的，而 Mac 的磁盘不是这个格式，因此就会导致二者之间无法拷贝文件。有一个解决方法就是使用 Tuxera NTFS For Mac，有了它，我们就可以比较顺利地拷贝文件了。 另外还有其他品牌的 NTFS For Mac 软件，也可以尝试使用一下。 推荐指数：★★★★☆</p>
                  <h3 id="VMware、Parallels-Desktop"><a href="#VMware、Parallels-Desktop" class="headerlink" title="VMware、Parallels Desktop"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#vmwareparallels-desktop" target="_blank" rel="noopener"></a>VMware、Parallels Desktop</h3>
                  <p>用了 Mac 之后，难免会有些情况下也还会不得不使用 Windows，毕竟很多软件可能只有 Windows 版本，但用 Mac 我就不推荐装双系统了，直接装虚拟机就好了，Mac 上虚拟机软件有两款比较好用，一个就是著名的 VMware，另一个就是 Parallels Desktop，这两款我都使用过，觉得都非常不错，现在用的是 VMware。 推荐指数：★★★★☆</p>
                  <h3 id="CleanMyMac"><a href="#CleanMyMac" class="headerlink" title="CleanMyMac"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#cleanmymac" target="_blank" rel="noopener"></a>CleanMyMac</h3>
                  <p>很多时候用着用着磁盘就不够用了，如果你的 Mac 硬盘是 512GB 的倒还好，256GB 的你就得多注意一下了，另外 1T 定制版土豪请绕道，这款软件不适合你。 CleanMyMac 可以非常方便地帮助我们扫描缓存、大文件、废纸篓、残留项等内容，清理这些内容之后我们可以节省很多硬盘空间，另外它还支持软件卸载和残留清扫功能，可以帮我们非常干净地移除 Mac 中的软件，目前应该是出到第三版了，非常推荐。 推荐指数：★★★★☆</p>
                  <h2 id="编辑器"><a href="#编辑器" class="headerlink" title="编辑器"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E7%BC%96%E8%BE%91%E5%99%A8" target="_blank" rel="noopener"></a>编辑器</h2>
                  <p>既然做程序开发嘛，不配置好自己的开发环境怎么行，下面推荐一下我平常使用的开发软件。</p>
                  <h3 id="JetBrains"><a href="#JetBrains" class="headerlink" title="JetBrains"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#jetbrains" target="_blank" rel="noopener"></a>JetBrains</h3>
                  <p>我目前使用的 IDE 是 JetBrains 全家桶，目前我编写 Python 比较多，所以主要使用 PyCharm，另外写前端的时候也会使用 WebStorm，写 Java 就用 IntelliJ IDEA，C、C++ 用 CLion，PHP 的话就用 PhpStorm，Ruby 的话就用 RubyMine，其他的语言用的就少了，就没有装了。 当然有的小伙伴会说 JetBrains 系列的 IDE 需要购买啊？我只想说，国人的力量是无穷的，在网上其实可以搜到各种破解方法，如 License Server 验证，你能搜到各种五花八门的 License Server。另外 JetBrains 还有专门的 Educational Programs，可以来这里申请：<a href="https://www.jetbrains.com/education/programs/?fromMenu" target="_blank" rel="noopener">https://www.jetbrains.com/education/programs/?fromMenu</a>，学生、老师或教育工作者可以使用学校的 edu 邮箱申请免费的 License，如果你还是学生的话，那么申请是十分方便的，因为我还是个学生，我目前就在使用学生套餐，当然如果你已经工作的话也可以向正在上学的弟弟妹妹们借一下嘛。 总之我个人比较喜欢 JetBrains 全家桶，不论是页面风格还是开发习惯我都比较喜欢，推荐使用。 推荐指数：★★★★☆</p>
                  <h3 id="Sublime"><a href="#Sublime" class="headerlink" title="Sublime"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#sublime" target="_blank" rel="noopener"></a>Sublime</h3>
                  <p>有时候我们可能下载了或接收了一些单个的文本文件，我们只想看看文本文件内容是什么，或者对其再做一些简单的修改操作，这时候就没必要单独用 JetBrains 的 IDE 打开了，显得有点重了。或者有时候需要修改某个配置文件，这时候也需要一个比较好用的编辑器。我使用的就是 Sublime，对于一些日常的文本编辑是足够了，另外 Sublime 还可以扩展好多插件，配置好了功能上基本不输 JetBrains IDE，非常推荐。 推荐指数：★★★★</p>
                  <h3 id="MarkEditor"><a href="#MarkEditor" class="headerlink" title="MarkEditor"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#markeditor" target="_blank" rel="noopener"></a>MarkEditor</h3>
                  <p>现在越来越多的写作平台开始支持 MarkDown，不得不说这确实是一门提高文字生产效率的语言，写 MarkDown 我强烈推荐 MarkEditor，我之前尝试过各种 MarkDown 写作软件，觉得都不如这款好用，如 Typora、MWeb、GitBook 等等。 MarkEditor 支持写作及预览模式，更重要的是支持文件管理，很多软件如 Typora 只能打开单个的 Makrdown 文件，不能打开整个文件夹，这就很鸡肋了。另外 MarkEditor 支持直接插入图片，如我们截了一张图或者刚从网上复制了一张图，在 MarkEditor 里面直接粘贴就可以了，它会自动把这张图保存到当前目录下，同时生成 Makrdown 格式的的图片链接，不能更方便了！另外还支持主题自定义、样式自定义，还可以快速插入某些 Makrdown 元素，还支持 Latex 公式，还可以快速导出电子书，快速生成文稿网页，快速局域网共享，功能应有尽有，强烈推荐！ 这个软件我购买了 Pro 版，解锁了全部功能，订购地址：<a href="https://www.markeditor.com/" target="_blank" rel="noopener">https://www.markeditor.com/</a>，个人觉得物超所值！ 推荐指数：★★★★★</p>
                  <h3 id="SnippetLab"><a href="#SnippetLab" class="headerlink" title="SnippetLab"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#snippetlab" target="_blank" rel="noopener"></a>SnippetLab</h3>
                  <p>在写代码的时候，我们经常会有一些常用代码或者精华代码，或者一些常用的配置，想要单独保存下来复用，这时我们可能会把它保存到某个文本文件里面，更高级点可以使用云笔记，如有道云笔记或者印象笔记，用过 GitHub Gists 的小伙伴可能会选择 GitHub Gists，但我觉得这些都不是最佳的。 首先文本文件、云笔记里面其实并不是专门为了保存代码使用的，另外 GitHub Gists 保存操作并没有那么便捷，而且打开速度也很慢，影响体验。在这里推荐一款专门用来保存代码的软件叫做 SnippetLab，涉设计初衷就是为了保存短代码片的，它支持几乎所有编程语言，另外支持分类、分级、加标签、加描述等，另外它还可以和 Alfred 对接实现快速搜索查找，另外还支持备份、导出、云同步等各种功能，非常适合做代码片的管理。 推荐指数：★★★★</p>
                  <h3 id="Beyond-Compare"><a href="#Beyond-Compare" class="headerlink" title="Beyond Compare"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#beyond-compare" target="_blank" rel="noopener"></a>Beyond Compare</h3>
                  <p>有时候我们需要比较两个文件的不同之处，以便于快速得知两个版本的修改内容，我使用的软件是 Beyond Compare，个人觉得比较简洁好用，同时删除和添加的内容有对应的红绿颜色标识，推荐给大家使用。 推荐指数：★★★☆</p>
                  <h2 id="管理工具"><a href="#管理工具" class="headerlink" title="管理工具"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E7%AE%A1%E7%90%86%E5%B7%A5%E5%85%B7" target="_blank" rel="noopener"></a>管理工具</h2>
                  <p>有时候我们需要管理很多文件，或者还需要远程管理很多终端设备，在这里推荐几款比较好用的工具。</p>
                  <h3 id="Filezlla"><a href="#Filezlla" class="headerlink" title="Filezlla"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#filezlla" target="_blank" rel="noopener"></a>Filezlla</h3>
                  <p>有时候我们需要管理一些远程的服务器，比如 Linux 服务器。那么如何和这些服务器之间传递数据和文件呢？这里推荐一个轻便简洁的软件 Filezlla，它支持 FTP、SFTP 等协议类型，使用它我们可以方便地进行文件传输和远程文件管理。 推荐指数：★★★</p>
                  <h3 id="ForkLift"><a href="#ForkLift" class="headerlink" title="ForkLift"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#forklift" target="_blank" rel="noopener"></a>ForkLift</h3>
                  <p>Mac 上的 Finder 你是不是已经受够了？在一些方面做得相当不友好，例如在当前打开的目录下新建一个空白文件，在当前的目录下打开命令行工具等等，有了 ForkLift 这些都是小意思了。另外 ForkLift 还集成了 Filezlla 的功能，利用它我们还可以像普通文件管理器一样管理远程的主机内容，它还支持 FTP、SFTP、SMB、WebDAV、NFS 等等各种协议。同时界面也非常美观，有了它，几乎可以抛弃 Finder 和 Filezlla 了，强烈推荐！ 推荐指数：★★★★☆</p>
                  <h3 id="SSH-Shell"><a href="#SSH-Shell" class="headerlink" title="SSH Shell"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#ssh-shell" target="_blank" rel="noopener"></a>SSH Shell</h3>
                  <p>我们经常会和各种服务器打交道，例如我们经常使用 SSH 来远程连接某台 Linux 服务器，原生 Terminal 是支持 SSH 的，但你会发现原生带的这个太难用了。可能很多小伙伴使用 iTerm，不得不说这确实是个神器，大大方便了远程管理流程。但我在这里还要推荐一个我经常使用的 SSH Shell，没错，它的名字就是 SSH Shell，它的页面操作简洁，同时管理和记录远程主机十分方便，另外还支持秘钥管理、自动重连、自定义主题等等功能，个人用起来十分顺手，强烈推荐！ 推荐指数：★★★★☆</p>
                  <h3 id="HomeBrew、CakeBrew"><a href="#HomeBrew、CakeBrew" class="headerlink" title="HomeBrew、CakeBrew"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#homebrewcakebrew" target="_blank" rel="noopener"></a>HomeBrew、CakeBrew</h3>
                  <p>对于开发者来说，这个软件几乎是 Mac 上必备的一个软件，它的官方简介就是 “The missing package manager for macOS”，算是 Mac 上的一个软件包平台，它里面包含着非常多的 Mac 开发软件包，比如 Python、PHP、Redis、MySQL、RabbitMQ、HBase 等等，几乎你能想到的开发软件都集成在里面了，堪称神器！ 它的安装也非常简单，参见这里：<a href="https://brew.sh/" target="_blank" rel="noopener">https://brew.sh/</a>，另外 HomeBrew 也有对应的图形界面，叫做 CakeBrew，如果不喜欢命令行操作的话可以使用 CakeBrew 来代替。 推荐指数：★★★★★</p>
                  <h2 id="影音图像"><a href="#影音图像" class="headerlink" title="影音图像"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E5%BD%B1%E9%9F%B3%E5%9B%BE%E5%83%8F" target="_blank" rel="noopener"></a>影音图像</h2>
                  <h3 id="IINA"><a href="#IINA" class="headerlink" title="IINA"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#iina" target="_blank" rel="noopener"></a>IINA</h3>
                  <p>这个必须要赞一下，非常强大简洁好用的视频播放器，是 GitHub 上的一个开源软件，链接是：<a href="https://lhc70000.github.io/iina/" target="_blank" rel="noopener">https://lhc70000.github.io/iina/</a>，播放控制、视频设置、音频设置、字幕设置、文件操作，几乎你能想到的应有尽有，而且无广告，简洁清爽，支持的视频格式也十分广泛，推荐使用！ 推荐指数：★★★★</p>
                  <h3 id="ScreenFlow"><a href="#ScreenFlow" class="headerlink" title="ScreenFlow"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#screenflow" target="_blank" rel="noopener"></a>ScreenFlow</h3>
                  <p>之前我曾录制过一些 Python 的视频课程，本来尝试过 QuickTime 录制，可是实在是太难用了，另外视频剪辑、音频剪辑等又是个麻烦事。后来我就使用了 ScreenFlow，它集录制、剪辑、配音、字幕、特效等功能于一体，另外录制质量，渲染质量也是一流，大大提高了我的效率，堪称神器！ 推荐指数：★★★★☆</p>
                  <h3 id="iPic"><a href="#iPic" class="headerlink" title="iPic"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#ipic" target="_blank" rel="noopener"></a>iPic</h3>
                  <p>有时候我们在写 MarkDown 的时候，可能突然需要一张插入一张图片，比如我们想插入一张屏幕截图，我们就需要把这张图片先存下来，然后加上图片的路径，如果转发给别人还需要连着图片一并发给对方，这其实是不怎么方便的，倘若这张图片是一张来自网络的图片，我们直接用 HTTP 访问的话，那岂不是方便太多了？ 要将图片传到网上分几步？三步。第一步，把上传页面打开，第二步，把图片传到网上并把传后链接拷贝下来，第三步，把上传页面关闭。简直是太麻烦了对不对？另外找个合适的图床也是个麻烦事啊，七牛？又拍？你不得又得申请和注册。那么有了 iPic，一切就不是难事了，它可以监听 Mac 的粘贴板，一旦我们复制了一张图或者新截了一张图，它就能显示到待上传队列里面，我们点一下它就会把图片上传到网络上，然后生成上传后的链接，默认使用的是新浪的图床，网速也非常快。有了它，传图什么的都不是事了！另外付费版还支持各种自定义图床，如七牛云、又拍云、阿里云、腾讯云等等。 推荐指数：★★★★☆</p>
                  <h3 id="PixelMator"><a href="#PixelMator" class="headerlink" title="PixelMator"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#pixelmator" target="_blank" rel="noopener"></a>PixelMator</h3>
                  <p>在 Windows 上我们常用 PS 来修改和处理图片，Mac 上我是没有使用 PS，使用了 PixelMator，个人觉得使用这款软件能完全胜任 PS 的工作，一般的图片设计、排版、抠图、特效、蒙版等操作都支持，我个人比较喜欢使用这款软件做设计。 推荐指数：★★★★</p>
                  <h3 id="Polarr-Photo-Editor"><a href="#Polarr-Photo-Editor" class="headerlink" title="Polarr Photo Editor"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#polarr-photo-editor" target="_blank" rel="noopener"></a>Polarr Photo Editor</h3>
                  <p>这个软件又名“泼辣修图”，类似 Mac 上的美图秀秀，它自带了各种后期滤镜，还带有 Lightroom 的很多调光调色的工具，能够帮我们快速对照片进行后期处理，效果也还不错，当然比不上 Photoshop 和 Lightroom 那么专业，但对于快速进行后处理的小伙伴来说不失为一个好的选择。 推荐指数：★★★★</p>
                  <h3 id="Boom2"><a href="#Boom2" class="headerlink" title="Boom2"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#boom2" target="_blank" rel="noopener"></a>Boom2</h3>
                  <p>我有边工作边听歌的习惯，所以音乐几乎离不开我的生活，入了个好耳机，那当然就得配上好音乐。大家肯定也听说过音效均衡器，我们可以调整不同的音效参数来达到不同的声音效果，如电子音、人声、环绕、重低音等等，在 Mac 上我觉得最好用的就是 Boom2 了，它内置了各种音效均衡器，还有一些高保真效果的渲染，效果非常给力。我一般听歌的时候就会把 Boom2 开起来，享受不一样的音效感觉，美哉。 推荐指数：★★★★</p>
                  <h2 id="趣味扩展"><a href="#趣味扩展" class="headerlink" title="趣味扩展"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#%E8%B6%A3%E5%91%B3%E6%89%A9%E5%B1%95" target="_blank" rel="noopener"></a>趣味扩展</h2>
                  <p>另外还有几个比较有意思的工具推荐下。</p>
                  <h3 id="Tickeys"><a href="#Tickeys" class="headerlink" title="Tickeys"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#tickeys" target="_blank" rel="noopener"></a>Tickeys</h3>
                  <p>使用过机械键盘吗？按键感觉和声音很爽吧，但是用了 Mac，你如果不使用外接键盘的话，想必手感就差上不少，但这款软件或许可以拯救一下，它可以模拟机械键盘的按键声，每次按键都有有机械键盘清脆的声音，我平时戴耳机撸代码的时候就会开着这个软件，感觉体验还是不错的，建议尝试一下。 推荐指数：★★★☆</p>
                  <h3 id="Duet"><a href="#Duet" class="headerlink" title="Duet"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#duet" target="_blank" rel="noopener"></a>Duet</h3>
                  <p>Duet 这款软件可以将 iPad 或 iPhone 变成电脑的扩展屏幕，如果你有一个大屏的比如 12.9 寸的 iPad 的话，非常建议你尝试一下这款软件，这样如果正你在用 Mac 不用 iPad 的话，完全可以用 Duet 把 iPad 和电脑屏幕连接起来来扩展显示，充分利用资源。 推荐指数：★★★☆ 好了，暂时推荐这么多，其实还有很多很多，尤其是专门针对于开发者的一些工具，这些就太偏极客化了，后面再为大家整理一些好用的开发者工具，敬请期待。 还不尽兴的小伙伴可以关注 GitHub 上的一个仓库叫 awesome-mac，里面列出来了 Mac 上推荐的非常多的软件，总结得非常非常详细，链接是：<a href="https://github.com/jaywcjlove/awesome-mac" target="_blank" rel="noopener">https://github.com/jaywcjlove/awesome-mac</a>，大家可以去看下。</p>
                  <h2 id="Tips"><a href="#Tips" class="headerlink" title="Tips"></a><a href="https://github.com/Germey/AI/blob/master/%E3%80%90Mac%E3%80%91Mac%E4%B8%8A%E6%8E%A8%E8%8D%90%E7%9A%84%E5%B7%A5%E5%85%B7%E9%9B%86%E5%90%88.md#tips" target="_blank" rel="noopener"></a>Tips</h2>
                  <p>可能有的小伙伴好奇我的 Launchpad 为啥能放那么多图标，是怎么做到的？其实很简单，几行代码就搞定了。 调整每列显示图标数量，这里以 7 为例：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">defaults <span class="keyword">write</span> com.apple.dock springboard-<span class="keyword">rows</span> -<span class="type">int</span> <span class="number">7</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>调整每行显示图标的数量，这里以 8 为例：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">defaults <span class="keyword">write</span> com.apple.dock springboard-<span class="keyword">columns</span> -<span class="type">int</span> <span class="number">8</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面两行代码最后的数字可以自行修改。 修改完了之后还需要重置一下 Launchpad，代码如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">defaults <span class="keyword">write</span> com.apple.dock ResetLaunchPad -<span class="type">bool</span> <span class="keyword">TRUE</span>;killall Dock</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好了，这样我们就可以自由定制我们的 Launchpad 图标数量啦！ 另外，还有的小伙伴会说，很多软件都需要花钱购买啊，咋办？告诉你个网址：<a href="http://xclient.info/" target="_blank" rel="noopener">http://xclient.info/</a>，几乎你想找的破解版都有，别说别的了，雷锋也别叫了，省下的钱打赏给我一点就行哈哈。 以上就是我的一些 Mac 常用软件分享及 Tips，希望对大家有帮助！ 另外大家如有还有推荐的软件，欢迎留言给我，非常感谢！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-07-03 02:35:47" itemprop="dateCreated datePublished" datetime="2018-07-03T02:35:47+08:00">2018-07-03</time>
                </span>
                <span id="/6146.html" class="post-meta-item leancloud_visitors" data-flag-title="推荐一些Mac上比较好用的软件" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6110.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6110.html" class="post-title-link" itemprop="url">《Python3网络爬虫开发实战》第二波抽奖赠书活动来了！</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>嗨~ 给大家重磅推荐一本书！上市两月就已经重印 4 次的 Python 爬虫书！它就是由静觅博客博主崔庆才所作的《Python3网络爬虫开发实战》！！！同时文末还有抽奖赠书活动，不容错过！！！</p>
                  <h2 id="书籍介绍"><a href="#书籍介绍" class="headerlink" title="书籍介绍"></a><strong>书籍介绍</strong></h2>
                  <p>本书<strong>《Python3网络爬虫开发实战》</strong>全面介绍了利用 Python3 开发网络爬虫的知识，书中首先详细介绍了各种类型的环境配置过程和爬虫基础知识，还讨论了 urllib、requests 等请求库和 Beautiful Soup、XPath、pyquery 等解析库以及文本和各类数据库的存储方法，另外本书通过多个真实新鲜案例介绍了分析 Ajax 进行数据爬取，Selenium 和 Splash 进行动态网站爬取的过程，接着又分享了一些切实可行的爬虫技巧，比如使用代理爬取和维护动态代理池的方法、ADSL 拨号代理的使用、各类验证码（图形、极验、点触、宫格等）的破解方法、模拟登录网站爬取的方法及 Cookies 池的维护等等。 此外，本书的内容还远远不止这些，作者还结合移动互联网的特点探讨了使用 Charles、mitmdump、Appium 等多种工具实现 App 抓包分析、加密参数接口爬取、微信朋友圈爬取的方法。此外本书还详细介绍了 pyspider 框架、Scrapy 框架的使用和分布式爬虫的知识，另外对于优化及部署工作，本书还包括 Bloom Filter 效率优化、Docker 和 Scrapyd 爬虫部署、分布式爬虫管理框架Gerapy 的分享。 全书共 604 页，足足两斤重呢~ 定价为 99 元！</p>
                  <h2 id="作者介绍"><a href="#作者介绍" class="headerlink" title="作者介绍"></a><strong>作者介绍</strong></h2>
                  <p>看书就先看看谁写的嘛，我们来了解一下~ 崔庆才，静觅博客博主（<a href="https://cuiqingcai.com），博客" target="_blank" rel="noopener">https://cuiqingcai.com），博客</a> Python 爬虫博文阅读量已过千万，北京航空航天大学硕士，天善智能、网易云课堂讲师，微软小冰大数据工程师，有多个大型分布式爬虫项目经验，乐于技术分享，文章通俗易懂 ^<em>^ 附皂片一张 ~(@^</em>^@)~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG785-240x320.jpeg" alt=""></p>
                  <h2 id="图文介绍"><a href="#图文介绍" class="headerlink" title="图文介绍"></a><strong>图文介绍</strong></h2>
                  <p>呕心沥血设计的宣传图也得放一下~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG556.jpeg" alt=""></p>
                  <h2 id="专家评论"><a href="#专家评论" class="headerlink" title="专家评论"></a><strong>专家评论</strong></h2>
                  <p>书是好是坏，得让专家看评一评呀，那么下面就是几位专家的精彩评论，快来看看吧~ 在互联网软件开发工程师的分类中，爬虫工程师是非常重要的。爬虫工作往往是一个公司核心业务开展的基础，数据抓取下来，才有后续的加工处理和最终展现。此时数据的抓取规模、稳定性、实时性、准确性就显得非常重要。早期的互联网充分开放互联，数据获取的难度很小。随着各大公司对数据资产日益看重，反爬水平也在不断提高，各种新技术不断给爬虫软件提出新的课题。本书作者对爬虫的各个领域都有深刻研究，书中探讨了Ajax数据的抓取、动态渲染页面的抓取、验证码识别、模拟登录等高级话题，同时也结合移动互联网的特点探讨了App的抓取等。更重要的是，本书提供了大量源码，可以帮助读者更好地理解相关内容。强烈推荐给各位技术爱好者阅读！</p>
                  <p><strong>——梁斌</strong>，八友科技总经理</p>
                  <p>数据既是当今大数据分析的前提，也是各种人工智能应用场景的基础。得数据者得天下，会爬虫者走遍天下也不怕！一册在手，让小白到老司机都能有所收获！</p>
                  <p><strong>——李舟军</strong>，北京航空航天大学教授，博士生导师</p>
                  <p>本书从爬虫入门到分布式抓取，详细介绍了爬虫技术的各个要点，并针对不同的场景提出了对应的解决方案。另外，书中通过大量的实例来帮助读者更好地学习爬虫技术，通俗易懂，干货满满。强烈推荐给大家！</p>
                  <p><strong>——宋睿华</strong>，微软小冰首席科学家</p>
                  <p>有人说中国互联网的带宽全给各种爬虫占据了，这说明网络爬虫的重要性以及中国互联网数据封闭垄断的现状。爬是一种能力，爬是为了不爬。</p>
                  <p><strong>——施水才</strong>，北京拓尔思信息技术股份有限公司总裁</p>
                  <h2 id="全书目录"><a href="#全书目录" class="headerlink" title="全书目录"></a><strong>全书目录</strong></h2>
                  <p>书的目录也有~ 看这里！</p>
                  <ul>
                    <li><strong>1-开发环境配置</strong></li>
                    <li>1.1-Python3的安装</li>
                    <li>1.2-请求库的安装</li>
                    <li>1.3-解析库的安装</li>
                    <li>1.4-数据库的安装</li>
                    <li>1.5-存储库的安装</li>
                    <li>1.6-Web库的安装</li>
                    <li>1.7-App爬取相关库的安装</li>
                    <li>1.8-爬虫框架的安装</li>
                    <li>1.9-部署相关库的安装</li>
                    <li><strong>2-爬虫基础</strong></li>
                    <li>2.1-HTTP基本原理</li>
                    <li>2.2-网页基础</li>
                    <li>2.3-爬虫的基本原理</li>
                    <li>2.4-会话和Cookies</li>
                    <li>2.5-代理的基本原理</li>
                    <li><strong>3-基本库的使用</strong></li>
                    <li>3.1-使用urllib</li>
                    <li>3.1.1-发送请求</li>
                    <li>3.1.2-处理异常</li>
                    <li>3.1.3-解析链接</li>
                    <li>3.1.4-分析Robots协议</li>
                    <li>3.2-使用requests</li>
                    <li>3.2.1-基本用法</li>
                    <li>3.2.2-高级用法</li>
                    <li>3.3-正则表达式</li>
                    <li>3.4-抓取猫眼电影排行</li>
                    <li><strong>4-解析库的使用</strong></li>
                    <li>4.1-使用XPath</li>
                    <li>4.2-使用Beautiful Soup</li>
                    <li>4.3-使用pyquery</li>
                    <li><strong>5-数据存储</strong></li>
                    <li>5.1-文件存储</li>
                    <li>5.1.1-TXT文本存储</li>
                    <li>5.1.2-JSON文件存储</li>
                    <li>5.1.3-CSV文件存储</li>
                    <li>5.2-关系型数据库存储</li>
                    <li>5.2.1-MySQL存储</li>
                    <li>5.3-非关系型数据库存储</li>
                    <li>5.3.1-MongoDB存储</li>
                    <li>5.3.2-Redis存储</li>
                    <li><strong>6-Ajax数据爬取</strong></li>
                    <li>6.1-什么是Ajax</li>
                    <li>6.2-Ajax分析方法</li>
                    <li>6.3-Ajax结果提取</li>
                    <li>6.4-分析Ajax爬取今日头条街拍美图</li>
                    <li><strong>7-动态渲染页面爬取</strong></li>
                    <li>7.1-Selenium的使用</li>
                    <li>7.2-Splash的使用</li>
                    <li>7.3-Splash负载均衡配置</li>
                    <li>7.4-使用Selenium爬取淘宝商品</li>
                    <li><strong>8-验证码的识别</strong></li>
                    <li>8.1-图形验证码的识别</li>
                    <li>8.2-极验滑动验证码的识别</li>
                    <li>8.3-点触验证码的识别</li>
                    <li>8.4-微博宫格验证码的识别</li>
                    <li><strong>9-代理的使用</strong></li>
                    <li>9.1-代理的设置</li>
                    <li>9.2-代理池的维护</li>
                    <li>9.3-付费代理的使用</li>
                    <li>9.4-ADSL拨号代理</li>
                    <li>9.5-使用代理爬取微信公众号文章</li>
                    <li><strong>10-模拟登录</strong></li>
                    <li>10.1-模拟登录并爬取GitHub</li>
                    <li>10.2-Cookies池的搭建</li>
                    <li><strong>11-App的爬取</strong></li>
                    <li>11.1-Charles的使用</li>
                    <li>11.2-mitmproxy的使用</li>
                    <li>11.3-mitmdump爬取“得到”App电子书信息</li>
                    <li>11.4-Appium的基本使用</li>
                    <li>11.5-Appium爬取微信朋友圈</li>
                    <li>11.6-Appium+mitmdump爬取京东商品</li>
                    <li><strong>12-pyspider框架的使用</strong></li>
                    <li>12.1-pyspider框架介绍</li>
                    <li>12.2-pyspider的基本使用</li>
                    <li>12.3-pyspider用法详解</li>
                    <li><strong>13-Scrapy框架的使用</strong></li>
                    <li>13.1-Scrapy框架介绍</li>
                    <li>13.2-Scrapy入门</li>
                    <li>13.3-Selector的用法</li>
                    <li>13.4-Spider的用法</li>
                    <li>13.5-Downloader Middleware的用法</li>
                    <li>13.6-Spider Middleware的用法</li>
                    <li>13.7-Item Pipeline的用法</li>
                    <li>13.8-Scrapy对接Selenium</li>
                    <li>13.9-Scrapy对接Splash</li>
                    <li>13.10-Scrapy通用爬虫</li>
                    <li>13.11-Scrapyrt的使用</li>
                    <li>13.12-Scrapy对接Docker</li>
                    <li>13.13-Scrapy爬取新浪微博</li>
                    <li><strong>14-分布式爬虫</strong></li>
                    <li>14.1-分布式爬虫原理</li>
                    <li>14.2-Scrapy-Redis源码解析</li>
                    <li>14.3-Scrapy分布式实现</li>
                    <li>14.4-Bloom Filter的对接</li>
                    <li><strong>15-分布式爬虫的部署</strong></li>
                    <li>15.1-Scrapyd分布式部署</li>
                    <li>15.2-Scrapyd-Client的使用</li>
                    <li>15.3-Scrapyd对接Docker</li>
                    <li>15.4-Scrapyd批量部署</li>
                    <li>15.5-Gerapy分布式管理</li>
                  </ul>
                  <h2 id="购买链接"><a href="#购买链接" class="headerlink" title="购买链接"></a><strong>购买链接</strong></h2>
                  <p>想必很多小伙伴已经等了很久了，之前预售那么久也一直迟迟没有货，发售就有不少网店又售空了，不过现在起不用担心了！</p>
                  <p>书籍现已在京东、天猫、当当等网店上架并全面供应啦，复制链接到浏览器打开或扫描二维码打开即可购买了！</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/京东二维码.png" alt=""></p>
                  <p> 京东商城</p>
                  <p><a href="https://item.jd.com/12333540.html" target="_blank" rel="noopener">https://item.jd.com/12333540.html</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/天猫二维码.png" alt=""></p>
                  <p> 天猫商城</p>
                  <p><a href="https://detail.tmall.com/item.htm?id=566699703917" target="_blank" rel="noopener">https://detail.tmall.com/item.htm?id=566699703917</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/当当二维码.png" alt=""></p>
                  <p>当当网</p>
                  <p><a href="http://product.dangdang.com/25249602.html" target="_blank" rel="noopener">http://product.dangdang.com/25249602.html</a></p>
                  <p>欢迎大家购买，谢谢支持！O(∩_∩)O</p>
                  <h2 id="免费预览"><a href="#免费预览" class="headerlink" title="免费预览"></a><strong>免费预览</strong></h2>
                  <p>不放心？想先看看有些啥，没问题！看这里： 免费章节试读（复制粘贴至浏览器打开）： <a href="https://cuiqingcai.com/5052.html">https://cuiqingcai.com/5052.html</a> 将一直免费开放<strong>前7章节</strong>，欢迎大家试读！ 好了，接下来就是我们的福利环节啦~</p>
                  <h2 id="福利一：抽奖送书！！！"><a href="#福利一：抽奖送书！！！" class="headerlink" title="福利一：抽奖送书！！！"></a><strong>福利一：抽奖送书！！！</strong></h2>
                  <p>恭喜你看到这里了！那么接下来的福利时间就到了！后面还有两个福利不容错过哦~ 抽奖送书活动第二波来袭（后面还有很多波哦），公众号抽奖送 30 本作者亲笔签名书籍！！！ 活动流程（重要，请一定认真阅读）： <strong>公众号进击的Coder回复 “抽奖” 获取抽奖码，2018.6.24 22:00 截止，逾期参与无效，请记住您的抽奖码，活动结束后会从参与活动的小伙伴中根据幸运值按照权重比例抽取 30 位并在微信公众号公布，届时请关注公众号抽奖结果的公布！获奖的小伙伴会获得作者亲笔签名的《Python3网络爬虫开发实战》一本。</strong></p>
                  <h2 id="福利二：独家优惠！！！"><a href="#福利二：独家优惠！！！" class="headerlink" title="福利二：独家优惠！！！"></a><strong>福利二：独家优惠！！！</strong></h2>
                  <p>等等，你以为这就是全部福利吗？当然不是！除了抽奖送书，我们还拿到了拨号VPS知名品牌云立方的独家优惠，在公众号（进击的Coder ）中回复：“优惠券”，即可免费领取云立方50元主机优惠券，数量有限，先到先得！优惠券可在云立方官网（www.yunlifang.cn）购买动态IP拨号VPS时抵扣现金，有了它，爬虫代理易如反掌！ 你问我动态拨号VPS能做什么？应该怎么用在爬虫里？来这里了解一下： <a href="https://mp.weixin.qq.com/s?__biz=MzIzNzA4NDk3Nw==&amp;mid=2457735734&amp;idx=1&amp;sn=ca0d066d767570205daa9475e31cbc1f&amp;scene=21#wechat_redirect" target="_blank" rel="noopener">轻松获得海量稳定代理！ADSL拨号代理的搭建</a></p>
                  <h2 id="福利三：视频课程！！！"><a href="#福利三：视频课程！！！" class="headerlink" title="福利三：视频课程！！！"></a><strong>福利三：视频课程！！！</strong></h2>
                  <p>当然除了书籍，也有配套的视频课程，作者同样是崔庆才，二者结合学习效果更佳！限时优惠折扣中！扫描下图中二维码即可了解详情！ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/课程宣传图.png" alt=""> 最后也是最重要的就是参与活动的地址了！！！快来扫码回复领取属于你的福利吧！！！</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/gzh.jpg" alt=""></p>
                  <h2 id="特别致谢"><a href="#特别致谢" class="headerlink" title="特别致谢"></a>特别致谢</h2>
                  <p>最后特别感谢云立方、天善智能对本活动的大力支持！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-06-12 01:42:51" itemprop="dateCreated datePublished" datetime="2018-06-12T01:42:51+08:00">2018-06-12</time>
                </span>
                <span id="/6110.html" class="post-meta-item leancloud_visitors" data-flag-title="《Python3网络爬虫开发实战》第二波抽奖赠书活动来了！" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6101.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6101.html" class="post-title-link" itemprop="url">自然语言处理中句子相似度计算的几种方法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在做自然语言处理的过程中，我们经常会遇到需要找出相似语句的场景，或者找出句子的近似表达，这时候我们就需要把类似的句子归到一起，这里面就涉及到句子相似度计算的问题，那么本节就来了解一下怎么样来用 Python 实现句子相似度的计算。</p>
                  <h2 id="基本方法"><a href="#基本方法" class="headerlink" title="基本方法"></a>基本方法</h2>
                  <p>句子相似度计算我们一共归类了以下几种方法：</p>
                  <ul>
                    <li>编辑距离计算</li>
                    <li>杰卡德系数计算</li>
                    <li>TF 计算</li>
                    <li>TFIDF 计算</li>
                    <li>Word2Vec 计算</li>
                  </ul>
                  <p>下面我们来一一了解一下这几种算法的原理和 Python 实现。</p>
                  <h2 id="编辑距离计算"><a href="#编辑距离计算" class="headerlink" title="编辑距离计算"></a>编辑距离计算</h2>
                  <p>编辑距离，英文叫做 Edit Distance，又称 Levenshtein 距离，是指两个字串之间，由一个转成另一个所需的最少编辑操作次数，如果它们的距离越大，说明它们越是不同。许可的编辑操作包括将一个字符替换成另一个字符，插入一个字符，删除一个字符。 例如我们有两个字符串：string 和 setting，如果我们想要把 string 转化为 setting，需要这么两步：</p>
                  <ul>
                    <li>第一步，在 s 和 t 之间加入字符 e。</li>
                    <li>第二步，把 r 替换成 t。</li>
                  </ul>
                  <p>所以它们的编辑距离差就是 2，这就对应着二者要进行转化所要改变（添加、替换、删除）的最小步数。 那么用 Python 怎样来实现呢，我们可以直接使用 distance 库：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import <span class="keyword">distance</span></span><br><span class="line"><span class="keyword"></span></span><br><span class="line"><span class="keyword">def </span>edit_distance(<span class="built_in">s1</span>, <span class="built_in">s2</span>):</span><br><span class="line">    return <span class="keyword">distance.levenshtein(s1, </span><span class="built_in">s2</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">s1</span> = <span class="string">'string'</span></span><br><span class="line"><span class="built_in">s2</span> = <span class="string">'setting'</span></span><br><span class="line">print(edit_distance(<span class="built_in">s1</span>, <span class="built_in">s2</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们直接使用 distance 库的 levenshtein() 方法，传入两个字符串，即可获取两个字符串的编辑距离了。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的 distance 库我们可以直接使用 pip3 来安装：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> distance</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样如果我们想要获取相似的文本的话可以直接设定一个编辑距离的阈值来实现，如设置编辑距离为 2，下面是一个样例：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> distance</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">edit_distance</span><span class="params">(s1, s2)</span>:</span></span><br><span class="line">    <span class="keyword">return</span> distance.levenshtein(s1, s2)</span><br><span class="line"></span><br><span class="line">strings = [</span><br><span class="line">    <span class="string">'你在干什么'</span>,</span><br><span class="line">    <span class="string">'你在干啥子'</span>,</span><br><span class="line">    <span class="string">'你在做什么'</span>,</span><br><span class="line">    <span class="string">'你好啊'</span>,</span><br><span class="line">    <span class="string">'我喜欢吃香蕉'</span></span><br><span class="line">]</span><br><span class="line"></span><br><span class="line">target = <span class="string">'你在干啥'</span></span><br><span class="line">results = list(filter(<span class="keyword">lambda</span> x: edit_distance(x, target) &lt;= <span class="number">2</span>, strings))</span><br><span class="line">print(results)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了一些字符串，然后定义了一个目标字符串，然后用编辑距离 2 的阈值进行设定，最后得到的结果就是编辑距离在 2 及以内的结果，运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'你在干什么</span>', <span class="symbol">'你在干啥子</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过这种方式我们可以大致筛选出类似的句子，但是发现一些句子例如“你在做什么” 就没有被识别出来，但他们的意义确实是相差不大的，因此，编辑距离并不是一个好的方式，但是简单易用。</p>
                  <h2 id="杰卡德系数计算"><a href="#杰卡德系数计算" class="headerlink" title="杰卡德系数计算"></a>杰卡德系数计算</h2>
                  <p>杰卡德系数，英文叫做 Jaccard index, 又称为 Jaccard 相似系数，用于比较有限样本集之间的相似性与差异性。Jaccard 系数值越大，样本相似度越高。 实际上它的计算方式非常简单，就是两个样本的交集除以并集得到的数值，当两个样本完全一致时，结果为 1，当两个样本完全不同时，结果为 0。 算法非常简单，就是交集除以并集，下面我们用 Python 代码来实现一下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> sklearn.feature_extraction.text <span class="keyword">import</span> CountVectorizer</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">jaccard_similarity</span><span class="params">(s1, s2)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">add_space</span><span class="params">(s)</span>:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">' '</span>.join(list(s))</span><br><span class="line">    </span><br><span class="line">    <span class="comment"># 将字中间加入空格</span></span><br><span class="line">    s1, s2 = add_space(s1), add_space(s2)</span><br><span class="line">    <span class="comment"># 转化为TF矩阵</span></span><br><span class="line">    cv = CountVectorizer(tokenizer=<span class="keyword">lambda</span> s: s.split())</span><br><span class="line">    corpus = [s1, s2]</span><br><span class="line">    vectors = cv.fit_transform(corpus).toarray()</span><br><span class="line">    <span class="comment"># 求交集</span></span><br><span class="line">    numerator = np.sum(np.min(vectors, axis=<span class="number">0</span>))</span><br><span class="line">    <span class="comment"># 求并集</span></span><br><span class="line">    denominator = np.sum(np.max(vectors, axis=<span class="number">0</span>))</span><br><span class="line">    <span class="comment"># 计算杰卡德系数</span></span><br><span class="line">    <span class="keyword">return</span> <span class="number">1.0</span> * numerator / denominator</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">s1 = <span class="string">'你在干嘛呢'</span></span><br><span class="line">s2 = <span class="string">'你在干什么呢'</span></span><br><span class="line">print(jaccard_similarity(s1, s2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用了 Sklearn 库中的 CountVectorizer 来计算句子的 TF 矩阵，然后利用 Numpy 来计算二者的交集和并集，随后计算杰卡德系数。 这里值得学习的有 CountVectorizer 的用法，通过它的 fit_transform() 方法我们可以将字符串转化为词频矩阵，例如这里有两句话“你在干嘛呢”和“你在干什么呢”，首先 CountVectorizer 会计算出不重复的有哪些字，会得到一个字的列表，结果为：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'么</span>', <span class="symbol">'什</span>', <span class="symbol">'你</span>', <span class="symbol">'呢</span>', <span class="symbol">'嘛</span>', <span class="symbol">'在</span>', <span class="symbol">'干</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个其实可以通过如下代码来获取，就是获取词表内容：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">cv</span><span class="selector-class">.get_feature_names</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来通过转化之后，vectors 变量就变成了：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[<span class="number">0</span> <span class="number">0</span> <span class="number">1</span> <span class="number">1</span> <span class="number">1</span> <span class="number">1</span> <span class="number">1</span>]</span><br><span class="line"> [<span class="number">1</span> <span class="number">1</span> <span class="number">1</span> <span class="number">1</span> <span class="number">0</span> <span class="number">1</span> <span class="number">1</span>]]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>它对应的是两个句子对应词表的词频统计，这里是两个句子，所以结果是一个长度为 2 的二维数组，比如第一句话“你在干嘛呢”中不包含“么”字，那么第一个“么”字对应的结果就是0，即数量为 0，依次类推。 后面我们使用了 np.min() 方法并传入了 axis 为 0，实际上就是获取了每一列的最小值，这样实际上就是取了交集，np.max() 方法是获取了每一列的最大值，实际上就是取了并集。 二者分别取和即是交集大小和并集大小，然后作商即可，结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">0.5714285714285714</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个数值越大，代表两个字符串越接近，否则反之，因此我们也可以使用这个方法，并通过设置一个相似度阈值来进行筛选。</p>
                  <h2 id="TF-计算"><a href="#TF-计算" class="headerlink" title="TF 计算"></a>TF 计算</h2>
                  <p>第三种方案就是直接计算 TF 矩阵中两个向量的相似度了，实际上就是求解两个向量夹角的余弦值，就是点乘积除以二者的模长，公式如下：</p>
                  <figure class="highlight gherkin">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cosθ=a·b/|<span class="string">a</span>|<span class="string">*</span>|<span class="string">b</span>|</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面我们已经获得了 TF 矩阵，下面我们只需要求解两个向量夹角的余弦值就好了，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> sklearn.feature_extraction.text <span class="keyword">import</span> CountVectorizer</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> scipy.linalg <span class="keyword">import</span> norm</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">tf_similarity</span><span class="params">(s1, s2)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">add_space</span><span class="params">(s)</span>:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">' '</span>.join(list(s))</span><br><span class="line">    </span><br><span class="line">    <span class="comment"># 将字中间加入空格</span></span><br><span class="line">    s1, s2 = add_space(s1), add_space(s2)</span><br><span class="line">    <span class="comment"># 转化为TF矩阵</span></span><br><span class="line">    cv = CountVectorizer(tokenizer=<span class="keyword">lambda</span> s: s.split())</span><br><span class="line">    corpus = [s1, s2]</span><br><span class="line">    vectors = cv.fit_transform(corpus).toarray()</span><br><span class="line">    <span class="comment"># 计算TF系数</span></span><br><span class="line">    <span class="keyword">return</span> np.dot(vectors[<span class="number">0</span>], vectors[<span class="number">1</span>]) / (norm(vectors[<span class="number">0</span>]) * norm(vectors[<span class="number">1</span>]))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">s1 = <span class="string">'你在干嘛呢'</span></span><br><span class="line">s2 = <span class="string">'你在干什么呢'</span></span><br><span class="line">print(tf_similarity(s1, s2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在在这里我们使用了 np.dot() 方法获取了向量的点乘积，然后通过 norm() 方法获取了向量的模长，经过计算得到二者的 TF 系数，结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">0.7302967433402214</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="TFIDF-计算"><a href="#TFIDF-计算" class="headerlink" title="TFIDF 计算"></a>TFIDF 计算</h2>
                  <p>另外除了计算 TF 系数我们还可以计算 TFIDF 系数，TFIDF 实际上就是在词频 TF 的基础上再加入 IDF 的信息，IDF 称为逆文档频率，不了解的可以看下阮一峰老师的讲解：<a href="http://www.ruanyifeng.com/blog/2013/03/tf-idf.html" target="_blank" rel="noopener">http://www.ruanyifeng.com/blog/2013/03/tf-idf.html</a>，里面对 TFIDF 的讲解也是十分透彻的。 下面我们还是借助于 Sklearn 中的模块 TfidfVectorizer 来实现，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> sklearn.feature_extraction.text <span class="keyword">import</span> TfidfVectorizer</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> scipy.linalg <span class="keyword">import</span> norm</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">tfidf_similarity</span><span class="params">(s1, s2)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">add_space</span><span class="params">(s)</span>:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">' '</span>.join(list(s))</span><br><span class="line">    </span><br><span class="line">    <span class="comment"># 将字中间加入空格</span></span><br><span class="line">    s1, s2 = add_space(s1), add_space(s2)</span><br><span class="line">    <span class="comment"># 转化为TF矩阵</span></span><br><span class="line">    cv = TfidfVectorizer(tokenizer=<span class="keyword">lambda</span> s: s.split())</span><br><span class="line">    corpus = [s1, s2]</span><br><span class="line">    vectors = cv.fit_transform(corpus).toarray()</span><br><span class="line">    <span class="comment"># 计算TF系数</span></span><br><span class="line">    <span class="keyword">return</span> np.dot(vectors[<span class="number">0</span>], vectors[<span class="number">1</span>]) / (norm(vectors[<span class="number">0</span>]) * norm(vectors[<span class="number">1</span>]))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">s1 = <span class="string">'你在干嘛呢'</span></span><br><span class="line">s2 = <span class="string">'你在干什么呢'</span></span><br><span class="line">print(tfidf_similarity(s1, s2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的 vectors 变量实际上就对应着 TFIDF 值，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[<span class="number">0.</span>         <span class="number">0.</span>         <span class="number">0.4090901</span>  <span class="number">0.4090901</span>  <span class="number">0.57496187</span> <span class="number">0.4090901</span> <span class="number">0.4090901</span> ]</span><br><span class="line"> [<span class="number">0.49844628</span> <span class="number">0.49844628</span> <span class="number">0.35464863</span> <span class="number">0.35464863</span> <span class="number">0.</span>  <span class="number">0.35464863</span> <span class="number">0.35464863</span>]]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">0.5803329846765686</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以通过 TFIDF 系数我们也可以进行相似度的计算。</p>
                  <h2 id="Word2Vec-计算"><a href="#Word2Vec-计算" class="headerlink" title="Word2Vec 计算"></a>Word2Vec 计算</h2>
                  <p>Word2Vec，顾名思义，其实就是将每一个词转换为向量的过程。如果不了解的话可以参考：<a href="https://blog.csdn.net/itplus/article/details/37969519" target="_blank" rel="noopener">https://blog.csdn.net/itplus/article/details/37969519</a>。 这里我们可以直接下载训练好的 Word2Vec 模型，模型的链接地址为：<a href="https://pan.baidu.com/s/1TZ8GII0CEX32ydjsfMc0zw" target="_blank" rel="noopener">https://pan.baidu.com/s/1TZ8GII0CEX32ydjsfMc0zw</a>，是使用新闻、百度百科、小说数据来训练的 64 维的 Word2Vec 模型，数据量很大，整体效果还不错，我们可以直接下载下来使用，这里我们使用的是 news_12g_baidubaike_20g_novel_90g_embedding_64.bin 数据，然后实现 Sentence2Vec，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> gensim</span><br><span class="line"><span class="keyword">import</span> jieba</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> scipy.linalg <span class="keyword">import</span> norm</span><br><span class="line"></span><br><span class="line">model_file = <span class="string">'./word2vec/news_12g_baidubaike_20g_novel_90g_embedding_64.bin'</span></span><br><span class="line">model = gensim.models.KeyedVectors.load_word2vec_format(model_file, binary=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">vector_similarity</span><span class="params">(s1, s2)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">sentence_vector</span><span class="params">(s)</span>:</span></span><br><span class="line">        words = jieba.lcut(s)</span><br><span class="line">        v = np.zeros(<span class="number">64</span>)</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> words:</span><br><span class="line">            v += model[word]</span><br><span class="line">        v /= len(words)</span><br><span class="line">        <span class="keyword">return</span> v</span><br><span class="line">    </span><br><span class="line">    v1, v2 = sentence_vector(s1), sentence_vector(s2)</span><br><span class="line">    <span class="keyword">return</span> np.dot(v1, v2) / (norm(v1) * norm(v2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在获取 Sentence Vector 的时候，我们首先对句子进行分词，然后对分好的每一个词获取其对应的 Vector，然后将所有 Vector 相加并求平均，这样就可得到 Sentence Vector 了，然后再计算其夹角余弦值即可。 调用示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">s1 = <span class="string">'你在干嘛'</span></span><br><span class="line">s2 = <span class="string">'你正做什么'</span></span><br><span class="line"><span class="function"><span class="title">vector_similarity</span><span class="params">(s1, s2)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">0.6701133967824016</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时如果我们再回到最初的例子看下效果：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">strings = [</span><br><span class="line">    <span class="string">'你在干什么'</span>,</span><br><span class="line">    <span class="string">'你在干啥子'</span>,</span><br><span class="line">    <span class="string">'你在做什么'</span>,</span><br><span class="line">    <span class="string">'你好啊'</span>,</span><br><span class="line">    <span class="string">'我喜欢吃香蕉'</span></span><br><span class="line">]</span><br><span class="line"></span><br><span class="line">target = <span class="string">'你在干啥'</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> <span class="built_in">string</span> in <span class="built_in">string</span><span class="variable">s:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="built_in">string</span>, vector_similarity(<span class="built_in">string</span>, target))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>依然是前面的例子，我们看下它们的匹配度结果是多少，运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">你在干什么 <span class="number">0.8785495016487204</span></span><br><span class="line">你在干啥子 <span class="number">0.9789649689827049</span></span><br><span class="line">你在做什么 <span class="number">0.8781992402695274</span></span><br><span class="line">你好啊 <span class="number">0.5174225914249863</span></span><br><span class="line">我喜欢吃香蕉 <span class="number">0.582990841450621</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到相近的语句相似度都能到 0.8 以上，而不同的句子相似度都不足 0.6，这个区分度就非常大了，可以说有了 Word2Vec 我们可以结合一些语义信息来进行一些判断，效果明显也好很多。 所以总体来说，Word2Vec 计算的方式是非常好的。 另外学术界还有一些可能更好的研究成果，这个可以参考知乎上的一些回答：<a href="https://www.zhihu.com/question/29978268/answer/54399062" target="_blank" rel="noopener">https://www.zhihu.com/question/29978268/answer/54399062</a>。 以上便是进行句子相似度计算的基本方法和 Python 实现，本节代码地址：<a href="https://github.com/AIDeepLearning/SentenceDistance" target="_blank" rel="noopener">https://github.com/AIDeepLearning/SentenceDistance</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-06-08 20:01:51" itemprop="dateCreated datePublished" datetime="2018-06-08T20:01:51+08:00">2018-06-08</time>
                </span>
                <span id="/6101.html" class="post-meta-item leancloud_visitors" data-flag-title="自然语言处理中句子相似度计算的几种方法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6058.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6058.html" class="post-title-link" itemprop="url">小白进阶之Scrapy第六篇Scrapy-Redis详解</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="Scrapy-Redis-详解"><a href="#Scrapy-Redis-详解" class="headerlink" title="Scrapy-Redis 详解"></a>Scrapy-Redis 详解</h1>
                  <p>通常我们在一个站站点进行采集的时候，如果是小站的话 我们使用scrapy本身就可以满足。 但是如果在面对一些比较大型的站点的时候，单个scrapy就显得力不从心了。 <img src="https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/6fd6b3659b0e7bc8c3ebcf741e221f3c.jpg" alt=""> 要是我们能够多个Scrapy一起采集该多好啊 人多力量大。 很遗憾Scrapy官方并不支持多个同时采集一个站点，虽然官方给出一个方法： <strong><strong>将一个站点的分割成几部分 交给不同的scrapy去采集</strong></strong> 似乎是个解决办法，但是很麻烦诶！毕竟分割很麻烦的哇 下面就改轮到我们的额主角Scrapy-Redis登场了！ <img src="https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/9af3afb195a78aa8770204115cc7959d.jpg" alt=""></p>
                  <h2 id="什么？？你这么就登场了？还没说为什么呢？"><a href="#什么？？你这么就登场了？还没说为什么呢？" class="headerlink" title="什么？？你这么就登场了？还没说为什么呢？"></a>什么？？你这么就登场了？还没说为什么呢？</h2>
                  <p>好吧 为了简单起见 就用官方图来简单说明一下： <img src="https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/62d1dc3038e8b596d6df6f8e8a28258a.jpg" alt=""> 这张图大家相信大家都很熟悉了。重点看一下SCHEDULER 1. 先来看看官方对于SCHEDULER的定义： <strong><strong>SCHEDULER接受来自Engine的Requests,并将它们放入队列（可以按顺序优先级），以便在之后将其提供给Engine</strong></strong> <a href="https://doc.scrapy.org/en/latest/topics/architecture.html#component-scheduler" target="_blank" rel="noopener">点我看文档</a> 2. 现在我们来看看SCHEDULER都提供了些什么功能： 根据官方文档说明 在我们没有没有指定 SCHEDULER 参数时，默认使用：’scrapy.core.scheduler.Scheduler’ 作为SCHEDULER(调度器) scrapy.core.scheduler.py: </p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Scheduler</span>(<span class="title">object</span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, dupefilter, jobdir=None, dqclass=None, mqclass=None,</span></span></span><br><span class="line"><span class="function"><span class="params">                 logunser=False, stats=None, pqclass=None)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.df = dupefilter</span><br><span class="line">        <span class="keyword">self</span>.dqdir = <span class="keyword">self</span>._dqdir(jobdir)</span><br><span class="line">        <span class="keyword">self</span>.pqclass = pqclass</span><br><span class="line">        <span class="keyword">self</span>.dqclass = dqclass</span><br><span class="line">        <span class="keyword">self</span>.mqclass = mqclass</span><br><span class="line">        <span class="keyword">self</span>.logunser = logunser</span><br><span class="line">        <span class="keyword">self</span>.stats = stats</span><br><span class="line">        <span class="comment"># 注意在scrpy中优先注意这个方法，此方法是一个钩子 用于访问当前爬虫的配置</span></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        settings = crawler.settings</span><br><span class="line">        <span class="comment"># 获取去重用的类 默认：scrapy.dupefilters.RFPDupeFilter</span></span><br><span class="line">        dupefilter_cls = load_object(settings[<span class="string">'DUPEFILTER_CLASS'</span>])</span><br><span class="line">        <span class="comment"># 对去重类进行配置from_settings 在 scrapy.dupefilters.RFPDupeFilter 43行</span></span><br><span class="line">        <span class="comment"># 这种调用方式对于IDE跳转不是很好  所以需要自己去找</span></span><br><span class="line">        <span class="comment"># <span class="doctag">@classmethod</span></span></span><br><span class="line">        <span class="comment"># def from_settings(cls, settings):</span></span><br><span class="line">        <span class="comment">#     debug = settings.getbool('DUPEFILTER_DEBUG')</span></span><br><span class="line">        <span class="comment">#     return cls(job_dir(settings), debug)</span></span><br><span class="line">        <span class="comment"># 上面就是from_settings方法 其实就是设置工作目录 和是否开启debug</span></span><br><span class="line">        dupefilter = dupefilter_cls.from_settings(settings)</span><br><span class="line">        <span class="comment"># 获取优先级队列 类对象 默认：queuelib.pqueue.PriorityQueue</span></span><br><span class="line">        pqclass = load_object(settings[<span class="string">'SCHEDULER_PRIORITY_QUEUE'</span>])</span><br><span class="line">        <span class="comment"># 获取磁盘队列 类对象（SCHEDULER使用磁盘存储 重启不会丢失）</span></span><br><span class="line">        dqclass = load_object(settings[<span class="string">'SCHEDULER_DISK_QUEUE'</span>])</span><br><span class="line">        <span class="comment"># 获取内存队列 类对象（SCHEDULER使用内存存储 重启会丢失）</span></span><br><span class="line">        mqclass = load_object(settings[<span class="string">'SCHEDULER_MEMORY_QUEUE'</span>])</span><br><span class="line">        <span class="comment"># 是否开启debug</span></span><br><span class="line">        logunser = settings.getbool(<span class="string">'LOG_UNSERIALIZABLE_REQUESTS'</span>, settings.getbool(<span class="string">'SCHEDULER_DEBUG'</span>))</span><br><span class="line">        <span class="comment"># 将这些参数传递给 __init__方法</span></span><br><span class="line">        <span class="keyword">return</span> cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,</span><br><span class="line">                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">has_pending_requests</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">      <span class="string">""</span><span class="string">"检查是否有没处理的请求"</span><span class="string">""</span></span><br><span class="line">        <span class="keyword">return</span> len(<span class="keyword">self</span>) &gt; <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">      <span class="string">""</span><span class="string">"Engine创建完毕之后会调用这个方法"</span><span class="string">""</span></span><br><span class="line">        <span class="keyword">self</span>.spider = spider</span><br><span class="line">        <span class="comment"># 创建一个有优先级的内存队列 实例化对象</span></span><br><span class="line">        <span class="comment"># self.pqclass 默认是：queuelib.pqueue.PriorityQueue</span></span><br><span class="line">        <span class="comment"># self._newmq 会返回一个内存队列的 实例化对象 在110  111 行</span></span><br><span class="line">        <span class="keyword">self</span>.mqs = <span class="keyword">self</span>.pqclass(<span class="keyword">self</span>._newmq)</span><br><span class="line">        <span class="comment"># 如果self.dqdir 有设置 就创建一个磁盘队列 否则self.dqs 为空</span></span><br><span class="line">        <span class="keyword">self</span>.dqs = <span class="keyword">self</span>._dq() <span class="keyword">if</span> <span class="keyword">self</span>.dqdir <span class="keyword">else</span> None</span><br><span class="line">        <span class="comment"># 获得一个去重实例对象 open 方法是从BaseDupeFilter继承的</span></span><br><span class="line">        <span class="comment"># 现在我们可以用self.df来去重啦</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.df.open()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close</span><span class="params">(<span class="keyword">self</span>, reason)</span></span><span class="symbol">:</span></span><br><span class="line">      <span class="string">""</span><span class="string">"当然Engine关闭时"</span><span class="string">""</span></span><br><span class="line">          <span class="comment"># 如果有磁盘队列 则对其进行dump后保存到active.json文件中</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">dqs:</span></span><br><span class="line">            prios = <span class="keyword">self</span>.dqs.close()</span><br><span class="line">            with open(join(<span class="keyword">self</span>.dqdir, <span class="string">'active.json'</span>), <span class="string">'w'</span>) as <span class="symbol">f:</span></span><br><span class="line">                json.dump(prios, f)</span><br><span class="line">        <span class="comment"># 然后关闭去重</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.df.close(reason)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">enqueue_request</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">      <span class="string">""</span><span class="string">"添加一个Requests进调度队列"</span><span class="string">""</span></span><br><span class="line">          <span class="comment"># self.df.request_seen是检查这个Request是否已经请求过了 如果有会返回True</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> request.dont_filter <span class="keyword">and</span> <span class="keyword">self</span>.df.request_seen(request)<span class="symbol">:</span></span><br><span class="line">              <span class="comment"># 如果Request的dont_filter属性没有设置（默认为False）和 已经存在则去重</span></span><br><span class="line">            <span class="comment"># 不push进队列</span></span><br><span class="line">            <span class="keyword">self</span>.df.log(request, <span class="keyword">self</span>.spider)</span><br><span class="line">            <span class="keyword">return</span> False</span><br><span class="line">        <span class="comment"># 先尝试将Request push进磁盘队列</span></span><br><span class="line">        dqok = <span class="keyword">self</span>._dqpush(request)</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">dqok:</span></span><br><span class="line">              <span class="comment"># 如果成功 则在记录一次状态</span></span><br><span class="line">            <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/enqueued/disk'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">              <span class="comment"># 不能添加进磁盘队列则会添加进内存队列</span></span><br><span class="line">            <span class="keyword">self</span>._mqpush(request)</span><br><span class="line">            <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/enqueued/memory'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/enqueued'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="keyword">return</span> True</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">next_request</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">      <span class="string">""</span><span class="string">"从队列中获取一个Request"</span><span class="string">""</span></span><br><span class="line">          <span class="comment"># 优先从内存队列中获取</span></span><br><span class="line">        request = <span class="keyword">self</span>.mqs.pop()</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">request:</span></span><br><span class="line">            <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/dequeued/memory'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">              <span class="comment"># 不能获取的时候从磁盘队列队里获取</span></span><br><span class="line">            request = <span class="keyword">self</span>._dqpop()</span><br><span class="line">            <span class="keyword">if</span> <span class="symbol">request:</span></span><br><span class="line">                <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/dequeued/disk'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">request:</span></span><br><span class="line">            <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/dequeued'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="comment"># 将获取的到Request返回给Engine</span></span><br><span class="line">        <span class="keyword">return</span> request</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> len(<span class="keyword">self</span>.dqs) + len(<span class="keyword">self</span>.mqs) <span class="keyword">if</span> <span class="keyword">self</span>.dqs <span class="keyword">else</span> len(<span class="keyword">self</span>.mqs)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_dqpush</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>.dqs is <span class="symbol">None:</span></span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        <span class="symbol">try:</span></span><br><span class="line">            reqd = request_to_dict(request, <span class="keyword">self</span>.spider)</span><br><span class="line">            <span class="keyword">self</span>.dqs.push(reqd, -request.priority)</span><br><span class="line">        except ValueError as <span class="symbol">e:</span>  <span class="comment"># non serializable request</span></span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">logunser:</span></span><br><span class="line">                msg = (<span class="string">"Unable to serialize request: %(request)s - reason:"</span></span><br><span class="line">                       <span class="string">" %(reason)s - no more unserializable requests will be"</span></span><br><span class="line">                       <span class="string">" logged (stats being collected)"</span>)</span><br><span class="line">                logger.warning(msg, &#123;<span class="string">'request'</span>: request, <span class="string">'reason'</span>: e&#125;,</span><br><span class="line">                               exc_info=True, extra=&#123;<span class="string">'spider'</span>: <span class="keyword">self</span>.spider&#125;)</span><br><span class="line">                <span class="keyword">self</span>.logunser = False</span><br><span class="line">            <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/unserializable'</span>,</span><br><span class="line">                                 spider=<span class="keyword">self</span>.spider)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">            <span class="keyword">return</span> True</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_mqpush</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mqs.push(request, -request.priority)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_dqpop</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">dqs:</span></span><br><span class="line">            d = <span class="keyword">self</span>.dqs.pop()</span><br><span class="line">            <span class="keyword">if</span> <span class="symbol">d:</span></span><br><span class="line">                <span class="keyword">return</span> request_from_dict(d, <span class="keyword">self</span>.spider)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_newmq</span><span class="params">(<span class="keyword">self</span>, priority)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.mqclass()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_newdq</span><span class="params">(<span class="keyword">self</span>, priority)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.dqclass(join(<span class="keyword">self</span>.dqdir, <span class="string">'p%s'</span> % priority))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_dq</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        activef = join(<span class="keyword">self</span>.dqdir, <span class="string">'active.json'</span>)</span><br><span class="line">        <span class="keyword">if</span> exists(activef)<span class="symbol">:</span></span><br><span class="line">            with open(activef) as <span class="symbol">f:</span></span><br><span class="line">                prios = json.load(f)</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">            prios = ()</span><br><span class="line">        q = <span class="keyword">self</span>.pqclass(<span class="keyword">self</span>._newdq, startprios=prios)</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">q:</span></span><br><span class="line">            logger.info(<span class="string">"Resuming crawl (%(queuesize)d requests scheduled)"</span>,</span><br><span class="line">                        &#123;<span class="string">'queuesize'</span>: len(q)&#125;, extra=&#123;<span class="string">'spider'</span>: <span class="keyword">self</span>.spider&#125;)</span><br><span class="line">        <span class="keyword">return</span> q</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_dqdir</span><span class="params">(<span class="keyword">self</span>, jobdir)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">jobdir:</span></span><br><span class="line">            dqdir = join(jobdir, <span class="string">'requests.queue'</span>)</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> exists(dqdir)<span class="symbol">:</span></span><br><span class="line">                os.makedirs(dqdir)</span><br><span class="line">            <span class="keyword">return</span> dqdir</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 只挑了一些重点的写了一些注释剩下大家自己领会(才不是我懒哦 ) 从上面的代码 我们可以很清楚的知道 SCHEDULER的主要是完成了 push Request pop Request 和 去重的操作。 而且queue 操作是在内存队列中完成的。 大家看queuelib.queue就会发现基于内存的（deque） 那么去重呢？</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RFPDupeFilter</span>(<span class="title">BaseDupeFilter</span>):</span></span><br><span class="line">    <span class="string">""</span><span class="string">"Request Fingerprint duplicates filter"</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, path=None, debug=False)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.file = None</span><br><span class="line">        <span class="keyword">self</span>.fingerprints = set()</span><br><span class="line">        <span class="keyword">self</span>.logdupes = True</span><br><span class="line">        <span class="keyword">self</span>.debug = debug</span><br><span class="line">        <span class="keyword">self</span>.logger = logging.getLogger(__name_<span class="number">_</span>)</span><br><span class="line">        <span class="keyword">if</span> <span class="symbol">path:</span></span><br><span class="line">              <span class="comment"># 此处可以看到去重其实打开了一个名叫 requests.seen的文件</span></span><br><span class="line">            <span class="comment"># 如果是使用的磁盘的话</span></span><br><span class="line">            <span class="keyword">self</span>.file = open(os.path.join(path, <span class="string">'requests.seen'</span>), <span class="string">'a+'</span>)</span><br><span class="line">            <span class="keyword">self</span>.file.seek(<span class="number">0</span>)</span><br><span class="line">            <span class="keyword">self</span>.fingerprints.update(x.rstrip() <span class="keyword">for</span> x <span class="keyword">in</span> <span class="keyword">self</span>.file)</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_settings</span><span class="params">(cls, settings)</span></span><span class="symbol">:</span></span><br><span class="line">        debug = settings.getbool(<span class="string">'DUPEFILTER_DEBUG'</span>)</span><br><span class="line">        <span class="keyword">return</span> cls(job_dir(settings), debug)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">request_seen</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">        fp = <span class="keyword">self</span>.request_fingerprint(request)</span><br><span class="line">        <span class="keyword">if</span> fp <span class="keyword">in</span> <span class="keyword">self</span>.<span class="symbol">fingerprints:</span></span><br><span class="line">              <span class="comment"># 判断我们的请求是否在这个在集合中</span></span><br><span class="line">            <span class="keyword">return</span> True</span><br><span class="line">        <span class="comment"># 没有在集合就添加进去</span></span><br><span class="line">        <span class="keyword">self</span>.fingerprints.add(fp)</span><br><span class="line">        <span class="comment"># 如果用的磁盘队列就写进去记录一下</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">file:</span></span><br><span class="line">            <span class="keyword">self</span>.file.write(fp + os.linesep)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <pre><code>  按照正常流程就是大家都会进行重复的采集；我们都知道进程之间内存中的数据不可共享的，那么你在开启多个Scrapy的时候，它们相互之间并不知道对方采集了些什么那些没有没采集。那就大家伙儿自己玩自己的了。完全没没有效率的提升啊！ ![](https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/7015cb643d0c05854dab5b8457f076af.jpg) 怎么解决呢？ 这就是我们Scrapy-Redis解决的问题了，不能协作不就是因为Request 和 去重这两个 不能共享吗？ 那我把这两个独立出来好了。 将Scrapy中的SCHEDULER组件独立放到大家都能访问的地方不就OK啦！加上scrapy-redis后流程图就应该变成这样了? ![](https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/0a94645a8f10707fe80610b5ebeb945e.jpg) So············· 这样是不是看起来就清楚多了？？？ 下面我们来看看Scrapy-Redis是怎么处理的? scrapy_redis.scheduler.py：
</code></pre>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">class Scheduler(object):</span><br><span class="line">    <span class="string">""</span><span class="string">"Redis-based scheduler</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Settings</span></span><br><span class="line"><span class="string">    --------</span></span><br><span class="line"><span class="string">    SCHEDULER_PERSIST : bool (default: False)</span></span><br><span class="line"><span class="string">        Whether to persist or clear redis queue.</span></span><br><span class="line"><span class="string">    SCHEDULER_FLUSH_ON_START : bool (default: False)</span></span><br><span class="line"><span class="string">        Whether to flush redis queue on start.</span></span><br><span class="line"><span class="string">    SCHEDULER_IDLE_BEFORE_CLOSE : int (default: 0)</span></span><br><span class="line"><span class="string">        How many seconds to wait before closing if no message is received.</span></span><br><span class="line"><span class="string">    SCHEDULER_QUEUE_KEY : str</span></span><br><span class="line"><span class="string">        Scheduler redis key.</span></span><br><span class="line"><span class="string">    SCHEDULER_QUEUE_CLASS : str</span></span><br><span class="line"><span class="string">        Scheduler queue class.</span></span><br><span class="line"><span class="string">    SCHEDULER_DUPEFILTER_KEY : str</span></span><br><span class="line"><span class="string">        Scheduler dupefilter redis key.</span></span><br><span class="line"><span class="string">    SCHEDULER_DUPEFILTER_CLASS : str</span></span><br><span class="line"><span class="string">        Scheduler dupefilter class.</span></span><br><span class="line"><span class="string">    SCHEDULER_SERIALIZER : str</span></span><br><span class="line"><span class="string">        Scheduler serializer.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line">    def __init__(self, server,</span><br><span class="line">                 <span class="attribute">persist</span>=<span class="literal">False</span>,</span><br><span class="line">                 <span class="attribute">flush_on_start</span>=<span class="literal">False</span>,</span><br><span class="line">                 <span class="attribute">queue_key</span>=defaults.SCHEDULER_QUEUE_KEY,</span><br><span class="line">                 <span class="attribute">queue_cls</span>=defaults.SCHEDULER_QUEUE_CLASS,</span><br><span class="line">                 <span class="attribute">dupefilter_key</span>=defaults.SCHEDULER_DUPEFILTER_KEY,</span><br><span class="line">                 <span class="attribute">dupefilter_cls</span>=defaults.SCHEDULER_DUPEFILTER_CLASS,</span><br><span class="line">                 <span class="attribute">idle_before_close</span>=0,</span><br><span class="line">                 <span class="attribute">serializer</span>=None):</span><br><span class="line">        <span class="string">""</span><span class="string">"Initialize scheduler.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        server : Redis</span></span><br><span class="line"><span class="string">            这是Redis实例</span></span><br><span class="line"><span class="string">        persist : bool</span></span><br><span class="line"><span class="string">            是否在关闭时清空Requests.默认值是False。</span></span><br><span class="line"><span class="string">        flush_on_start : bool</span></span><br><span class="line"><span class="string">            是否在启动时清空Requests。 默认值是False。</span></span><br><span class="line"><span class="string">        queue_key : str</span></span><br><span class="line"><span class="string">            Request队列的Key名字</span></span><br><span class="line"><span class="string">        queue_cls : str</span></span><br><span class="line"><span class="string">            队列的可导入路径（就是使用什么队列）</span></span><br><span class="line"><span class="string">        dupefilter_key : str</span></span><br><span class="line"><span class="string">            去重队列的Key</span></span><br><span class="line"><span class="string">        dupefilter_cls : str</span></span><br><span class="line"><span class="string">            去重类的可导入路径。</span></span><br><span class="line"><span class="string">        idle_before_close : int</span></span><br><span class="line"><span class="string">            等待多久关闭</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        <span class="keyword">if</span> idle_before_close &lt; 0:</span><br><span class="line">            raise TypeError(<span class="string">"idle_before_close cannot be negative"</span>)</span><br><span class="line"></span><br><span class="line">        self.server = server</span><br><span class="line">        self.persist = persist</span><br><span class="line">        self.flush_on_start = flush_on_start</span><br><span class="line">        self.queue_key = queue_key</span><br><span class="line">        self.queue_cls = queue_cls</span><br><span class="line">        self.dupefilter_cls = dupefilter_cls</span><br><span class="line">        self.dupefilter_key = dupefilter_key</span><br><span class="line">        self.idle_before_close = idle_before_close</span><br><span class="line">        self.serializer = serializer</span><br><span class="line">        self.stats = None</span><br><span class="line"></span><br><span class="line">    def __len__(self):</span><br><span class="line">        return len(self.queue)</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def from_settings(cls, settings):</span><br><span class="line">        kwargs = &#123;</span><br><span class="line">            <span class="string">'persist'</span>: settings.getbool(<span class="string">'SCHEDULER_PERSIST'</span>),</span><br><span class="line">            <span class="string">'flush_on_start'</span>: settings.getbool(<span class="string">'SCHEDULER_FLUSH_ON_START'</span>),</span><br><span class="line">            <span class="string">'idle_before_close'</span>: settings.getint(<span class="string">'SCHEDULER_IDLE_BEFORE_CLOSE'</span>),</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        # <span class="keyword">If</span> these values are missing, it means we want <span class="keyword">to</span> use the defaults.</span><br><span class="line">        optional = &#123;</span><br><span class="line">            # TODO: Use custom prefixes <span class="keyword">for</span> this<span class="built_in"> settings </span><span class="keyword">to</span><span class="built_in"> note </span>that are</span><br><span class="line">            # specific <span class="keyword">to</span> scrapy-redis.</span><br><span class="line">            <span class="string">'queue_key'</span>: <span class="string">'SCHEDULER_QUEUE_KEY'</span>,</span><br><span class="line">            <span class="string">'queue_cls'</span>: <span class="string">'SCHEDULER_QUEUE_CLASS'</span>,</span><br><span class="line">            <span class="string">'dupefilter_key'</span>: <span class="string">'SCHEDULER_DUPEFILTER_KEY'</span>,</span><br><span class="line">            # We use the<span class="built_in"> default </span>setting name <span class="keyword">to</span> keep compatibility.</span><br><span class="line">            <span class="string">'dupefilter_cls'</span>: <span class="string">'DUPEFILTER_CLASS'</span>,</span><br><span class="line">            <span class="string">'serializer'</span>: <span class="string">'SCHEDULER_SERIALIZER'</span>,</span><br><span class="line">        &#125;</span><br><span class="line">        # 从setting中获取配置组装成dict(具体获取那些配置是optional字典中key)</span><br><span class="line">        <span class="keyword">for</span> name, setting_name <span class="keyword">in</span> optional.items():</span><br><span class="line">            val = settings.<span class="builtin-name">get</span>(setting_name)</span><br><span class="line">            <span class="keyword">if</span> val:</span><br><span class="line">                kwargs[name] = val</span><br><span class="line"></span><br><span class="line">        # Support serializer as a path <span class="keyword">to</span> a module.</span><br><span class="line">        <span class="keyword">if</span> isinstance(kwargs.<span class="builtin-name">get</span>(<span class="string">'serializer'</span>), six.string_types):</span><br><span class="line">            kwargs[<span class="string">'serializer'</span>] = importlib.import_module(kwargs[<span class="string">'serializer'</span>])</span><br><span class="line">                # 或得一个Redis连接</span><br><span class="line">       <span class="built_in"> server </span>= connection.from_settings(settings)</span><br><span class="line">        # Ensure the<span class="built_in"> connection </span>is working.</span><br><span class="line">        server.ping()</span><br><span class="line"></span><br><span class="line">        return cls(<span class="attribute">server</span>=server, **kwargs)</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def from_crawler(cls, crawler):</span><br><span class="line">       <span class="built_in"> instance </span>= cls.from_settings(crawler.settings)</span><br><span class="line">        # FIXME: <span class="keyword">for</span> now, stats are only supported <span class="keyword">from</span> this constructor</span><br><span class="line">        instance.stats = crawler.stats</span><br><span class="line">        return instance</span><br><span class="line"></span><br><span class="line">    def open(self, spider):</span><br><span class="line">        self.spider = spider</span><br><span class="line"></span><br><span class="line">        try:</span><br><span class="line">              # 根据self.queue_cls这个可以导入的类 实例化一个队列</span><br><span class="line">            self.queue = load_object(self.queue_cls)(</span><br><span class="line">                <span class="attribute">server</span>=self.server,</span><br><span class="line">                <span class="attribute">spider</span>=spider,</span><br><span class="line">                <span class="attribute">key</span>=self.queue_key % &#123;<span class="string">'spider'</span>: spider.name&#125;,</span><br><span class="line">                <span class="attribute">serializer</span>=self.serializer,</span><br><span class="line">            )</span><br><span class="line">        except TypeError as e:</span><br><span class="line">            raise ValueError(<span class="string">"Failed to instantiate queue class '%s': %s"</span>,</span><br><span class="line">                             self.queue_cls, e)</span><br><span class="line"></span><br><span class="line">        try:</span><br><span class="line">              # 根据self.dupefilter_cls这个可以导入的类 实例一个去重集合</span><br><span class="line">            # 默认是集合 可以实现自己的去重方式 比如 bool 去重</span><br><span class="line">            self.df = load_object(self.dupefilter_cls)(</span><br><span class="line">                <span class="attribute">server</span>=self.server,</span><br><span class="line">                <span class="attribute">key</span>=self.dupefilter_key % &#123;<span class="string">'spider'</span>: spider.name&#125;,</span><br><span class="line">                <span class="attribute">debug</span>=spider.settings.getbool('DUPEFILTER_DEBUG'),</span><br><span class="line">            )</span><br><span class="line">        except TypeError as e:</span><br><span class="line">            raise ValueError(<span class="string">"Failed to instantiate dupefilter class '%s': %s"</span>,</span><br><span class="line">                             self.dupefilter_cls, e)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.flush_on_start:</span><br><span class="line">            self.flush()</span><br><span class="line">        # notice <span class="keyword">if</span> there are requests already <span class="keyword">in</span> the<span class="built_in"> queue </span><span class="keyword">to</span> resume the crawl</span><br><span class="line">        <span class="keyword">if</span> len(self.queue):</span><br><span class="line">            spider.log(<span class="string">"Resuming crawl (%d requests scheduled)"</span> % len(self.queue))</span><br><span class="line"></span><br><span class="line">    def close(self, reason):</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> self.persist:</span><br><span class="line">            self.flush()</span><br><span class="line"></span><br><span class="line">    def flush(self):</span><br><span class="line">        self.df.clear()</span><br><span class="line">        self.queue.clear()</span><br><span class="line"></span><br><span class="line">    def enqueue_request(self, request):</span><br><span class="line">      <span class="string">""</span><span class="string">"这个和Scrapy本身的一样"</span><span class="string">""</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> request.dont_filter <span class="keyword">and</span> self.df.request_seen(request):</span><br><span class="line">            self.df.log(request, self.spider)</span><br><span class="line">            return <span class="literal">False</span></span><br><span class="line">        <span class="keyword">if</span> self.stats:</span><br><span class="line">            self.stats.inc_value(<span class="string">'scheduler/enqueued/redis'</span>, <span class="attribute">spider</span>=self.spider)</span><br><span class="line">        # 向队列里面添加一个Request</span><br><span class="line">        self.queue.push(request)</span><br><span class="line">        return <span class="literal">True</span></span><br><span class="line"></span><br><span class="line">    def next_request(self):</span><br><span class="line">      <span class="string">""</span><span class="string">"获取一个Request"</span><span class="string">""</span></span><br><span class="line">        block_pop_timeout = self.idle_before_close</span><br><span class="line">        # block_pop_timeout 是一个等待参数 队列没有东西会等待这个时间  超时就会关闭</span><br><span class="line">        request = self.queue.pop(block_pop_timeout)</span><br><span class="line">        <span class="keyword">if</span> request <span class="keyword">and</span> self.stats:</span><br><span class="line">            self.stats.inc_value(<span class="string">'scheduler/dequeued/redis'</span>, <span class="attribute">spider</span>=self.spider)</span><br><span class="line">        return request</span><br><span class="line"></span><br><span class="line">    def has_pending_requests(self):</span><br><span class="line">        return len(self) &gt; 0</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 来先来看看 以上就是Scrapy-Redis中的SCHEDULER模块。下面我们来看看queue和本身的什么不同： scrapy_redis.queue.py 以最常用的优先级队列 PriorityQueue 举例： </p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PriorityQueue</span><span class="params">(Base)</span>:</span></span><br><span class="line">    <span class="string">"""Per-spider priority queue abstraction using redis' sorted set"""</span></span><br><span class="line">        <span class="string">"""其实就是使用Redis的有序集合 来对Request进行排序，这样就可以优先级高的在有序集合的顶层 我们只需要"""</span></span><br><span class="line">    <span class="string">"""从上往下依次获取Request即可"""</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Return the length of the queue"""</span></span><br><span class="line">        <span class="keyword">return</span> self.server.zcard(self.key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">push</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Push a request"""</span></span><br><span class="line">        <span class="string">"""添加一个Request进队列"""</span></span><br><span class="line">        <span class="comment"># self._encode_request 将Request请求进行序列化</span></span><br><span class="line">        data = self._encode_request(request)</span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        d = &#123;</span></span><br><span class="line"><span class="string">        'url': to_unicode(request.url),  # urls should be safe (safe_string_url)</span></span><br><span class="line"><span class="string">        'callback': cb,</span></span><br><span class="line"><span class="string">        'errback': eb,</span></span><br><span class="line"><span class="string">        'method': request.method,</span></span><br><span class="line"><span class="string">        'headers': dict(request.headers),</span></span><br><span class="line"><span class="string">        'body': request.body,</span></span><br><span class="line"><span class="string">        'cookies': request.cookies,</span></span><br><span class="line"><span class="string">        'meta': request.meta,</span></span><br><span class="line"><span class="string">        '_encoding': request._encoding,</span></span><br><span class="line"><span class="string">        'priority': request.priority,</span></span><br><span class="line"><span class="string">        'dont_filter': request.dont_filter,</span></span><br><span class="line"><span class="string">        'flags': request.flags,</span></span><br><span class="line"><span class="string">        '_class': request.__module__ + '.' + request.__class__.__name__</span></span><br><span class="line"><span class="string">            &#125;</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        data就是上面这个字典的序列化</span></span><br><span class="line"><span class="string">        在Scrapy.utils.reqser.py 中的request_to_dict方法中处理</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># 在Redis有序集合中数值越小优先级越高(就是会被放在顶层)所以这个位置是取得 相反数</span></span><br><span class="line">        score = -request.priority</span><br><span class="line">        <span class="comment"># We don't use zadd method as the order of arguments change depending on</span></span><br><span class="line">        <span class="comment"># whether the class is Redis or StrictRedis, and the option of using</span></span><br><span class="line">        <span class="comment"># kwargs only accepts strings, not bytes.</span></span><br><span class="line">        <span class="comment"># ZADD 是添加进有序集合</span></span><br><span class="line">        self.server.execute_command(<span class="string">'ZADD'</span>, self.key, score, data)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pop</span><span class="params">(self, timeout=<span class="number">0</span>)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        Pop a request</span></span><br><span class="line"><span class="string">        timeout not support in this queue class</span></span><br><span class="line"><span class="string">        有序集合不支持超时所以就木有使用timeout了  这个timeout就是挂羊头卖狗肉</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="string">"""从有序集合中取出一个Request"""</span></span><br><span class="line">        <span class="comment"># use atomic range/remove using multi/exec</span></span><br><span class="line">        <span class="string">"""使用multi的原因是为了将获取Request和删除Request合并成一个操作(原子性的)在获取到一个元素之后 删除它，因为有序集合 不像list 有pop 这种方式啊"""</span></span><br><span class="line">        pipe = self.server.pipeline()</span><br><span class="line">        pipe.multi()</span><br><span class="line">        <span class="comment"># 取出 顶层第一个</span></span><br><span class="line">        <span class="comment"># zrange :返回有序集 key 中，指定区间内的成员。0,0 就是第一个了</span></span><br><span class="line">        <span class="comment"># zremrangebyrank：移除有序集 key 中，指定排名(rank)区间内的所有成员 0，0也就是第一个了</span></span><br><span class="line">        <span class="comment"># 更多请参考Redis官方文档</span></span><br><span class="line">        pipe.zrange(self.key, <span class="number">0</span>, <span class="number">0</span>).zremrangebyrank(self.key, <span class="number">0</span>, <span class="number">0</span>)</span><br><span class="line">        results, count = pipe.execute()</span><br><span class="line">        <span class="keyword">if</span> results:</span><br><span class="line">            <span class="keyword">return</span> self._decode_request(results[<span class="number">0</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 以上就是SCHEDULER在处理Request的时候做的操作了。 是时候来看看SCHEDULER是怎么处理去重的了！ 只需要注意这个?方法即可：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">request_seen</span><span class="params">(self, request)</span>:</span></span><br><span class="line">  <span class="string">"""Returns True if request was already seen.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        request : scrapy.http.Request</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        Returns</span></span><br><span class="line"><span class="string">        -------</span></span><br><span class="line"><span class="string">        bool</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">  <span class="comment"># 通过self.request_fingerprint 会生一个sha1的指纹</span></span><br><span class="line">  fp = self.request_fingerprint(request)</span><br><span class="line">  <span class="comment"># This returns the number of values added, zero if already exists.</span></span><br><span class="line">  <span class="comment"># 添加进一个Redis集合如果self.key这个集合中存在fp这个指纹会返回1  不存在返回0</span></span><br><span class="line">  added = self.server.sadd(self.key, fp)</span><br><span class="line">  <span class="keyword">return</span> added == <span class="number">0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <pre><code>这样大家就都可以访问同一个Redis 获取同一个spider的Request 在同一个位置去重，就不用担心重复啦 大概就像这样：
</code></pre>
                  <ol>
                    <li>spider1：检查一下这个Request是否在Redis去重，如果在就证明其它的spider采集过啦！如果不在就添加进调度队列，等待别 人获取。自己继续干活抓取网页 产生新的Request了 重复之前步骤。</li>
                    <li>spider2：以相同的逻辑执行</li>
                  </ol>
                  <p>可能有些小伙儿会产生疑问了~~！spider2拿到了别人的Request了 怎么能正确的执行呢？逻辑不会错吗？ 这个不用担心啦 因为整Request当中包含了，所有的逻辑，回去看看上面那个序列化的字典。 总结一下：</p>
                  <ol>
                    <li>1. Scrapy-Reids 就是将Scrapy原本在内存中处理的 调度(就是一个队列Queue)、去重、这两个操作通过Redis来实现</li>
                    <li>多个Scrapy在采集同一个站点时会使用相同的redis key（可以理解为队列）添加Request 获取Request 去重Request，这样所有的spider不会进行重复采集。效率自然就嗖嗖的上去了。</li>
                    <li>3. Redis是原子性的，好处不言而喻(一个Request要么被处理 要么没被处理，不存在第三可能)</li>
                  </ol>
                  <p>另外Scrapy-Redis本身不支持Redis-Cluster，大量网站去重的话会给单机很大的压力（就算使用boolfilter 内存也不够整啊！） 改造方式很简单：</p>
                  <ol>
                    <li>使用 <strong><strong>rediscluster</strong></strong> 这个包替换掉本身的Redis连接</li>
                    <li>Redis-Cluster 不支持事务，可以使用lua脚本进行代替（lua脚本是原子性的哦）</li>
                    <li><strong><strong>注意使用lua脚本 不能写占用时间很长的操作</strong></strong>（毕竟一大群人等着操作Redis 你总不能让人家等着吧）</li>
                  </ol>
                  <p>以上！完毕 对于懒人小伙伴儿 看看这个我改好的: <a href="https://github.com/thsheep/scrapy_redis_cluster" target="_blank" rel="noopener">集群版Scrapy-Redis</a> <strong><strong>PS: 支持Python3.6+ 哦 ！ 其余的版本没测试过</strong></strong> <img src="https://thsheep-wordpress.oss-cn-beijing.aliyuncs.com/4a99cdf73bef5f5aaaa4ec9f61b8d838.jpg" alt=""></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-06-06 09:10:29" itemprop="dateCreated datePublished" datetime="2018-06-06T09:10:29+08:00">2018-06-06</time>
                </span>
                <span id="/6058.html" class="post-meta-item leancloud_visitors" data-flag-title="小白进阶之Scrapy第六篇Scrapy-Redis详解" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>14k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>13 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6080.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6080.html" class="post-title-link" itemprop="url">Python中logging模块的基本用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在 PyCon 2018 上，Mario Corchero 介绍了在开发过程中如何更方便轻松地记录日志的流程。</p>
                  <p><img src="https://user-gold-cdn.xitu.io/2018/6/3/163c61861faf256f?imageView2/0/w/1280/h/960/format/webp/ignore-error/1" alt=""></p>
                  <p>整个演讲的内容包括：</p>
                  <ul>
                    <li>为什么日志记录非常重要</li>
                    <li>日志记录的流程是怎样的</li>
                    <li>怎样来进行日志记录</li>
                    <li>怎样进行日志记录相关配置</li>
                    <li>日志记录使用常见误区</li>
                  </ul>
                  <p>下面我们来梳理一下整个演讲的过程，其实其核心就是介绍了 logging 模块的使用方法和一些配置。</p>
                  <h2 id="日志记录的重要性"><a href="#日志记录的重要性" class="headerlink" title="日志记录的重要性"></a>日志记录的重要性</h2>
                  <p>在开发过程中，如果程序运行出现了问题，我们是可以使用我们自己的 Debug 工具来检测到到底是哪一步出现了问题，如果出现了问题的话，是很容易排查的。但程序开发完成之后，我们会将它部署到生产环境中去，这时候代码相当于是在一个黑盒环境下运行的，我们只能看到其运行的效果，是不能直接看到代码运行过程中每一步的状态的。在这个环境下，运行过程中难免会在某个地方出现问题，甚至这个问题可能是我们开发过程中未曾遇到的问题，碰到这种情况应该怎么办？ 如果我们现在只能得知当前问题的现象，而没有其他任何信息的话，如果我们想要解决掉这个问题的话，那么只能根据问题的现象来试图复现一下，然后再一步步去调试，这恐怕是很难的，很大的概率上我们是无法精准地复现这个问题的，而且 Debug 的过程也会耗费巨多的时间，这样一旦生产环境上出现了问题，修复就会变得非常棘手。但这如果我们当时有做日志记录的话，不论是正常运行还是出现报错，都有相关的时间记录，状态记录，错误记录等，那么这样我们就可以方便地追踪到在当时的运行过程中出现了怎样的状况，从而可以快速排查问题。 因此，日志记录是非常有必要的，任何一款软件如果没有标准的日志记录，都不能算作一个合格的软件。作为开发者，我们需要重视并做好日志记录过程。</p>
                  <h2 id="日志记录的流程框架"><a href="#日志记录的流程框架" class="headerlink" title="日志记录的流程框架"></a>日志记录的流程框架</h2>
                  <p>那么在 Python 中，怎样才能算作一个比较标准的日志记录过程呢？或许很多人会使用 print 语句输出一些运行信息，然后再在控制台观察，运行的时候再将输出重定向到文件输出流保存到文件中，这样其实是非常不规范的，在 Python 中有一个标准的 logging 模块，我们可以使用它来进行标注的日志记录，利用它我们可以更方便地进行日志记录，同时还可以做更方便的级别区分以及一些额外日志信息的记录，如时间、运行模块信息等。 接下来我们先了解一下日志记录流程的整体框架。 <img src="https://user-gold-cdn.xitu.io/2018/6/3/163c618941cfc03f?imageView2/0/w/1280/h/960/format/webp/ignore-error/1" alt=""> 如图所示，整个日志记录的框架可以分为这么几个部分：</p>
                  <ul>
                    <li>Logger：即 Logger Main Class，是我们进行日志记录时创建的对象，我们可以调用它的方法传入日志模板和信息，来生成一条条日志记录，称作 Log Record。</li>
                    <li>Log Record：就代指生成的一条条日志记录。</li>
                    <li>Handler：即用来处理日志记录的类，它可以将 Log Record 输出到我们指定的日志位置和存储形式等，如我们可以指定将日志通过 FTP 协议记录到远程的服务器上，Handler 就会帮我们完成这些事情。</li>
                    <li>Formatter：实际上生成的 Log Record 也是一个个对象，那么我们想要把它们保存成一条条我们想要的日志文本的话，就需要有一个格式化的过程，那么这个过程就由 Formatter 来完成，返回的就是日志字符串，然后传回给 Handler 来处理。</li>
                    <li>Filter：另外保存日志的时候我们可能不需要全部保存，我们可能只需要保存我们想要的部分就可以了，所以保存前还需要进行一下过滤，留下我们想要的日志，如只保存某个级别的日志，或只保存包含某个关键字的日志等，那么这个过滤过程就交给 Filter 来完成。</li>
                    <li>Parent Handler：Handler 之间可以存在分层关系，以使得不同 Handler 之间共享相同功能的代码。</li>
                  </ul>
                  <p>以上就是整个 logging 模块的基本架构和对象功能，了解了之后我们详细来了解一下 logging 模块的用法。</p>
                  <h2 id="日志记录的相关用法"><a href="#日志记录的相关用法" class="headerlink" title="日志记录的相关用法"></a>日志记录的相关用法</h2>
                  <p>总的来说 logging 模块相比 print 有这么几个优点：</p>
                  <ul>
                    <li>可以在 logging 模块中设置日志等级，在不同的版本（如开发环境、生产环境）上通过设置不同的输出等级来记录对应的日志，非常灵活。</li>
                    <li>print 的输出信息都会输出到标准输出流中，而 logging 模块就更加灵活，可以设置输出到任意位置，如写入文件、写入远程服务器等。</li>
                    <li>logging 模块具有灵活的配置和格式化功能，如配置输出当前模块信息、运行时间等，相比 print 的字符串格式化更加方便易用。</li>
                  </ul>
                  <p>下面我们初步来了解下 logging 模块的基本用法，先用一个实例来感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logging.basicConfig(<span class="attribute">level</span>=logging.INFO, <span class="attribute">format</span>=<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line"></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'This is a log info'</span>)</span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finish'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们首先引入了 logging 模块，然后进行了一下基本的配置，这里通过 basicConfig 配置了 level 信息和 format 信息，这里 level 配置为 INFO 信息，即只输出 INFO 级别的信息，另外这里指定了 format 格式的字符串，包括 asctime、name、levelname、message 四个内容，分别代表运行时间、模块名称、日志级别、日志内容，这样输出内容便是这四者组合而成的内容了，这就是 logging 的全局配置。 接下来声明了一个 Logger 对象，它就是日志输出的主类，调用对象的 info() 方法就可以输出 INFO 级别的日志信息，调用 debug() 方法就可以输出 DEBUG 级别的日志信息，非常方便。在初始化的时候我们传入了模块的名称，这里直接使用 <strong>name</strong> 来代替了，就是模块的名称，如果直接运行这个脚本的话就是 <strong>main</strong>，如果是 import 的模块的话就是被引入模块的名称，这个变量在不同的模块中的名字是不同的，所以一般使用 <strong>name</strong> 来表示就好了，再接下来输出了四条日志信息，其中有两条 INFO、一条 WARNING、一条 DEBUG 信息，我们看下输出结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">42</span>:<span class="number">43</span>,<span class="number">526</span> - __main__ - INFO - This <span class="keyword">is</span> a log info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">42</span>:<span class="number">43</span>,<span class="number">526</span> - __main__ - WARNING - Warning exists</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">42</span>:<span class="number">43</span>,<span class="number">526</span> - __main__ - INFO - Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到输出结果一共有三条日志信息，每条日志都是对应了指定的格式化内容，另外我们发现 DEBUG 的信息是没有输出的，这是因为我们在全局配置的时候设置了输出为 INFO 级别，所以 DEBUG 级别的信息就被过滤掉了。 这时如果我们将输出的日志级别设置为 DEBUG，就可以看到 DEBUG 级别的日志输出了：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">logging.basic<span class="constructor">Config(<span class="params">level</span>=<span class="params">logging</span>.DEBUG, <span class="params">format</span>='%(<span class="params">asctime</span>)</span>s - %(name)s - %(levelname)s - %(message)s')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">49</span>:<span class="number">22</span>,<span class="number">770</span> - __main__ - INFO - This <span class="keyword">is</span> a log info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">49</span>:<span class="number">22</span>,<span class="number">770</span> - __main__ - DEBUG - Debugging</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">49</span>:<span class="number">22</span>,<span class="number">770</span> - __main__ - WARNING - Warning exists</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">13</span>:<span class="number">49</span>:<span class="number">22</span>,<span class="number">770</span> - __main__ - INFO - Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>由此可见，相比 print 来说，通过刚才的代码，我们既可以输出时间、模块名称，又可以输出不同级别的日志信息作区分并加以过滤，是不是灵活多了？ 当然这只是 logging 模块的一小部分功能，接下来我们首先来全面了解一下 basicConfig 的参数都有哪些：</p>
                  <ul>
                    <li>filename：即日志输出的文件名，如果指定了这个信息之后，实际上会启用 FileHandler，而不再是 StreamHandler，这样日志信息便会输出到文件中了。</li>
                    <li>filemode：这个是指定日志文件的写入方式，有两种形式，一种是 w，一种是 a，分别代表清除后写入和追加写入。</li>
                    <li>format：指定日志信息的输出格式，即上文示例所示的参数，详细参数可以参考：<a href="https://link.juejin.im?target=https%3A%2F%2Fdocs.python.org%2F3%2Flibrary%2Flogging.html%3Fhighlight%3Dlogging%2520threadname%23logrecord-attributes">docs.python.org/3/library/l…</a>，部分参数如下所示：<ul>
                        <li>%(levelno)s：打印日志级别的数值。</li>
                        <li>%(levelname)s：打印日志级别的名称。</li>
                        <li>%(pathname)s：打印当前执行程序的路径，其实就是sys.argv[0]。</li>
                        <li>%(filename)s：打印当前执行程序名。</li>
                        <li>%(funcName)s：打印日志的当前函数。</li>
                        <li>%(lineno)d：打印日志的当前行号。</li>
                        <li>%(asctime)s：打印日志的时间。</li>
                        <li>%(thread)d：打印线程ID。</li>
                        <li>%(threadName)s：打印线程名称。</li>
                        <li>%(process)d：打印进程ID。</li>
                        <li>%(processName)s：打印线程名称。</li>
                        <li>%(module)s：打印模块名称。</li>
                        <li>%(message)s：打印日志信息。</li>
                      </ul>
                    </li>
                    <li>datefmt：指定时间的输出格式。</li>
                    <li>style：如果 format 参数指定了，这个参数就可以指定格式化时的占位符风格，如 %、{、$ 等。</li>
                    <li>level：指定日志输出的类别，程序会输出大于等于此级别的信息。</li>
                    <li>stream：在没有指定 filename 的时候会默认使用 StreamHandler，这时 stream 可以指定初始化的文件流。</li>
                    <li>handlers：可以指定日志处理时所使用的 Handlers，必须是可迭代的。</li>
                  </ul>
                  <p>下面我们再用一个实例来感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logging.basicConfig(<span class="attribute">level</span>=logging.DEBUG,</span><br><span class="line">                    <span class="attribute">filename</span>=<span class="string">'output.log'</span>,</span><br><span class="line">                    <span class="attribute">datefmt</span>=<span class="string">'%Y/%m/%d %H:%M:%S'</span>,</span><br><span class="line">                    <span class="attribute">format</span>=<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(lineno)d - %(module)s - %(message)s'</span>)</span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line"></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'This is a log info'</span>)</span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finish'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们指定了输出文件的名称为 output.log，另外指定了日期的输出格式，其中年月日的格式变成了 %Y/%m/%d，另外输出的 format 格式增加了 lineno、module 这两个信息，运行之后便会生成一个 output.log 的文件，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">14</span>:<span class="number">43</span>:<span class="number">26</span> - __main__ - INFO - <span class="number">9</span> - demo3 - This <span class="keyword">is</span> a log info</span><br><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">14</span>:<span class="number">43</span>:<span class="number">26</span> - __main__ - DEBUG - <span class="number">10</span> - demo3 - Debugging</span><br><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">14</span>:<span class="number">43</span>:<span class="number">26</span> - __main__ - WARNING - <span class="number">11</span> - demo3 - Warning exists</span><br><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">14</span>:<span class="number">43</span>:<span class="number">26</span> - __main__ - INFO - <span class="number">12</span> - demo3 - Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到日志便会输出到文件中，同时输出了行号、模块名称等信息。 以上我们通过 basicConfig 来进行了一些全局的配置，我们同样可以使用 Formatter、Handler 进行更灵活的处理，下面我们来了解一下。</p>
                  <h3 id="Level"><a href="#Level" class="headerlink" title="Level"></a>Level</h3>
                  <p>首先我们来了解一下输出日志的等级信息，logging 模块共提供了如下等级，每个等级其实都对应了一个数值，列表如下：</p>
                  <p>等级</p>
                  <p>数值</p>
                  <p>CRITICAL</p>
                  <p>50</p>
                  <p>FATAL</p>
                  <p>50</p>
                  <p>ERROR</p>
                  <p>40</p>
                  <p>WARNING</p>
                  <p>30</p>
                  <p>WARN</p>
                  <p>30</p>
                  <p>INFO</p>
                  <p>20</p>
                  <p>DEBUG</p>
                  <p>10</p>
                  <p>NOTSET</p>
                  <p>0</p>
                  <p>这里最高的等级是 CRITICAL 和 FATAL，两个对应的数值都是 50，另外对于 WARNING 还提供了简写形式 WARN，两个对应的数值都是 30。 我们设置了输出 level，系统便只会输出 level 数值大于或等于该 level 的的日志结果，例如我们设置了输出日志 level 为 INFO，那么输出级别大于等于 INFO 的日志，如 WARNING、ERROR 等，DEBUG 和 NOSET 级别的不会输出。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.WARN)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Log</span></span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.critical(<span class="string">'Critical Something'</span>)</span><br><span class="line">logger.<span class="builtin-name">error</span>(<span class="string">'Error Occurred'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finished'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们设置了输出级别为 WARN，然后对应输出了五种不同级别的日志信息，运行结果如下：</p>
                  <figure class="highlight subunit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Critical Something</span><br><span class="line"><span class="keyword">Error </span>Occurred</span><br><span class="line">Warning exists</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到只有 CRITICAL、ERROR、WARNING 信息输出了，DEBUG、INFO 信息没有输出。</p>
                  <h3 id="Handler"><a href="#Handler" class="headerlink" title="Handler"></a>Handler</h3>
                  <p>下面我们先来了解一下 Handler 的用法，看下面的实例：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.INFO)</span><br><span class="line">handler = logging.FileHandler(<span class="string">'output.log'</span>)</span><br><span class="line">formatter = logging.Formatter(<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line">handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(handler)</span><br><span class="line"></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'This is a log info'</span>)</span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finish'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们没有再使用 basicConfig 全局配置，而是先声明了一个 Logger 对象，然后指定了其对应的 Handler 为 FileHandler 对象，然后 Handler 对象还单独指定了 Formatter 对象单独配置输出格式，最后给 Logger 对象添加对应的 Handler 即可，最后可以发现日志就会被输出到 output.log 中，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">14</span>:<span class="number">53</span>:<span class="number">36</span>,<span class="number">467</span> - __main__ - INFO - This <span class="keyword">is</span> a log info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">14</span>:<span class="number">53</span>:<span class="number">36</span>,<span class="number">468</span> - __main__ - WARNING - Warning exists</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">14</span>:<span class="number">53</span>:<span class="number">36</span>,<span class="number">468</span> - __main__ - INFO - Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外我们还可以使用其他的 Handler 进行日志的输出，logging 模块提供的 Handler 有：</p>
                  <ul>
                    <li>StreamHandler：logging.StreamHandler；日志输出到流，可以是 sys.stderr，sys.stdout 或者文件。</li>
                    <li>FileHandler：logging.FileHandler；日志输出到文件。</li>
                    <li>BaseRotatingHandler：logging.handlers.BaseRotatingHandler；基本的日志回滚方式。</li>
                    <li>RotatingHandler：logging.handlers.RotatingHandler；日志回滚方式，支持日志文件最大数量和日志文件回滚。</li>
                    <li>TimeRotatingHandler：logging.handlers.TimeRotatingHandler；日志回滚方式，在一定时间区域内回滚日志文件。</li>
                    <li>SocketHandler：logging.handlers.SocketHandler；远程输出日志到TCP/IP sockets。</li>
                    <li>DatagramHandler：logging.handlers.DatagramHandler；远程输出日志到UDP sockets。</li>
                    <li>SMTPHandler：logging.handlers.SMTPHandler；远程输出日志到邮件地址。</li>
                    <li>SysLogHandler：logging.handlers.SysLogHandler；日志输出到syslog。</li>
                    <li>NTEventLogHandler：logging.handlers.NTEventLogHandler；远程输出日志到Windows NT/2000/XP的事件日志。</li>
                    <li>MemoryHandler：logging.handlers.MemoryHandler；日志输出到内存中的指定buffer。</li>
                    <li>HTTPHandler：logging.handlers.HTTPHandler；通过”GET”或者”POST”远程输出到HTTP服务器。</li>
                  </ul>
                  <p>下面我们使用三个 Handler 来实现日志同时输出到控制台、文件、HTTP 服务器：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"><span class="keyword">from</span> logging.handlers import HTTPHandler</span><br><span class="line">import sys</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.DEBUG)</span><br><span class="line"></span><br><span class="line"><span class="comment"># StreamHandler</span></span><br><span class="line">stream_handler = logging.StreamHandler(sys.stdout)</span><br><span class="line">stream_handler.setLevel(<span class="attribute">level</span>=logging.DEBUG)</span><br><span class="line">logger.addHandler(stream_handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># FileHandler</span></span><br><span class="line">file_handler = logging.FileHandler(<span class="string">'output.log'</span>)</span><br><span class="line">file_handler.setLevel(<span class="attribute">level</span>=logging.INFO)</span><br><span class="line">formatter = logging.Formatter(<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line">file_handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(file_handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># HTTPHandler</span></span><br><span class="line">http_handler = HTTPHandler(<span class="attribute">host</span>=<span class="string">'localhost:8001'</span>, <span class="attribute">url</span>=<span class="string">'log'</span>, <span class="attribute">method</span>=<span class="string">'POST'</span>)</span><br><span class="line">logger.addHandler(http_handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Log</span></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'This is a log info'</span>)</span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finish'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之前我们需要先启动 HTTP Server，并运行在 8001 端口，其中 log 接口是用来接收日志的接口。 运行之后控制台输出会输出如下内容：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">This <span class="keyword">is</span> a <span class="keyword">log</span> <span class="keyword">info</span></span><br><span class="line">Debugging</span><br><span class="line"><span class="built_in">Warning</span> <span class="keyword">exists</span></span><br><span class="line">Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>output.log 文件会写入如下内容：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">15</span>:<span class="number">13</span>:<span class="number">44</span>,<span class="number">895</span> - __main__ - INFO - This <span class="keyword">is</span> a log info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">15</span>:<span class="number">13</span>:<span class="number">44</span>,<span class="number">947</span> - __main__ - WARNING - Warning exists</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">15</span>:<span class="number">13</span>:<span class="number">44</span>,<span class="number">949</span> - __main__ - INFO - Finish</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>HTTP Server 会收到控制台输出的信息。 这样一来，我们就通过设置多个 Handler 来控制了日志的多目标输出。 另外值得注意的是，在这里 StreamHandler 对象我们没有设置 Formatter，因此控制台只输出了日志的内容，而没有包含时间、模块等信息，而 FileHandler 我们通过 setFormatter() 方法设置了一个 Formatter 对象，因此输出的内容便是格式化后的日志信息。 另外每个 Handler 还可以设置 level 信息，最终输出结果的 level 信息会取 Logger 对象的 level 和 Handler 对象的 level 的交集。</p>
                  <h3 id="Formatter"><a href="#Formatter" class="headerlink" title="Formatter"></a>Formatter</h3>
                  <p>在进行日志格式化输出的时候，我们可以不借助于 basicConfig 来全局配置格式化输出内容，可以借助于 Formatter 来完成，下面我们再来单独看下 Formatter 的用法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.WARN)</span><br><span class="line">formatter = logging.Formatter(<span class="attribute">fmt</span>=<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>, <span class="attribute">datefmt</span>=<span class="string">'%Y/%m/%d %H:%M:%S'</span>)</span><br><span class="line">handler = logging.StreamHandler()</span><br><span class="line">handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Log</span></span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Debugging'</span>)</span><br><span class="line">logger.critical(<span class="string">'Critical Something'</span>)</span><br><span class="line">logger.<span class="builtin-name">error</span>(<span class="string">'Error Occurred'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Warning exists'</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finished'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们指定了一个 Formatter，并传入了 fmt 和 datefmt 参数，这样就指定了日志结果的输出格式和时间格式，然后 handler 通过 setFormatter() 方法设置此 Formatter 对象即可，输出结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">15</span>:<span class="number">47</span>:<span class="number">15</span> - __main__ - CRITICAL - Critical Something</span><br><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">15</span>:<span class="number">47</span>:<span class="number">15</span> - __main__ - ERROR - Error Occurred</span><br><span class="line"><span class="number">2018</span>/<span class="number">06</span>/<span class="number">03</span> <span class="number">15</span>:<span class="number">47</span>:<span class="number">15</span> - __main__ - WARNING - Warning exists</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们可以每个 Handler 单独配置输出的格式，非常灵活。</p>
                  <h3 id="捕获-Traceback"><a href="#捕获-Traceback" class="headerlink" title="捕获 Traceback"></a>捕获 Traceback</h3>
                  <p>如果遇到错误，我们更希望报错时出现的详细 Traceback 信息，便于调试，利用 logging 模块我们可以非常方便地实现这个记录，我们用一个实例来感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(__name__)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.DEBUG)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Formatter</span></span><br><span class="line">formatter = logging.Formatter(<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># FileHandler</span></span><br><span class="line">file_handler = logging.FileHandler(<span class="string">'result.log'</span>)</span><br><span class="line">file_handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(file_handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># StreamHandler</span></span><br><span class="line">stream_handler = logging.StreamHandler()</span><br><span class="line">stream_handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(stream_handler)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Log</span></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Start'</span>)</span><br><span class="line">logger.<span class="builtin-name">warning</span>(<span class="string">'Something maybe fail.'</span>)</span><br><span class="line">try:</span><br><span class="line">    result = 10 / 0</span><br><span class="line">except Exception:</span><br><span class="line">    logger.<span class="builtin-name">error</span>(<span class="string">'Faild to get result'</span>, <span class="attribute">exc_info</span>=<span class="literal">True</span>)</span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Finished'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们在 error() 方法中添加了一个参数，将 exc_info 设置为了 True，这样我们就可以输出执行过程中的信息了，即完整的 Traceback 信息。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">00</span>:<span class="number">15</span>,<span class="number">382</span> - __main__ - INFO - Start print log</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">00</span>:<span class="number">15</span>,<span class="number">382</span> - __main__ - DEBUG - Do something</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">00</span>:<span class="number">15</span>,<span class="number">382</span> - __main__ - WARNING - Something maybe fail.</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">00</span>:<span class="number">15</span>,<span class="number">382</span> - __main__ - ERROR - Faild to <span class="keyword">get</span> result</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File <span class="string">"/private/var/books/aicodes/loggingtest/demo8.py"</span>, line <span class="number">23</span>, <span class="keyword">in</span> &lt;module&gt;</span><br><span class="line">    result = <span class="number">10</span> / <span class="number">0</span></span><br><span class="line">ZeroDivisionError: division by zero</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">00</span>:<span class="number">15</span>,<span class="number">383</span> - __main__ - INFO - Finished</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这样我们就非常方便地记录下来了报错的信息，一旦出现了错误，我们也能非常方便地排查。</p>
                  <h3 id="配置共享"><a href="#配置共享" class="headerlink" title="配置共享"></a>配置共享</h3>
                  <p>在写项目的时候，我们肯定会将许多配置放置在许多模块下面，这时如果我们每个文件都来配置 logging 配置那就太繁琐了，logging 模块提供了父子模块共享配置的机制，会根据 Logger 的名称来自动加载父模块的配置。 例如我们这里首先定义一个 main.py 文件：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line">import core</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(<span class="string">'main'</span>)</span><br><span class="line">logger.setLevel(<span class="attribute">level</span>=logging.DEBUG)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Handler</span></span><br><span class="line">handler = logging.FileHandler(<span class="string">'result.log'</span>)</span><br><span class="line">handler.setLevel(logging.INFO)</span><br><span class="line">formatter = logging.Formatter(<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line">handler.setFormatter(formatter)</span><br><span class="line">logger.addHandler(handler)</span><br><span class="line"></span><br><span class="line">logger.<span class="builtin-name">info</span>(<span class="string">'Main Info'</span>)</span><br><span class="line">logger.<span class="builtin-name">debug</span>(<span class="string">'Main Debug'</span>)</span><br><span class="line">logger.<span class="builtin-name">error</span>(<span class="string">'Main Error'</span>)</span><br><span class="line">core.<span class="builtin-name">run</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们配置了日志的输出格式和文件路径，同时定义了 Logger 的名称为 main，然后引入了另外一个模块 core，最后调用了 core 的 run() 方法。 接下来我们定义 core.py，内容如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(<span class="string">'main.core'</span>)</span><br><span class="line"></span><br><span class="line">def <span class="builtin-name">run</span>():</span><br><span class="line">    logger.<span class="builtin-name">info</span>(<span class="string">'Core Info'</span>)</span><br><span class="line">    logger.<span class="builtin-name">debug</span>(<span class="string">'Core Debug'</span>)</span><br><span class="line">    logger.<span class="builtin-name">error</span>(<span class="string">'Core Error'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了 Logger 的名称为 main.core，注意这里开头是 main，即刚才我们在 main.py 里面的 Logger 的名称，这样 core.py 里面的 Logger 就会复用 main.py 里面的 Logger 配置，而不用再去配置一次了。 运行之后会生成一个 result.log 文件，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">55</span>:<span class="number">56</span>,<span class="number">259</span> - main - INFO - Main Info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">55</span>:<span class="number">56</span>,<span class="number">259</span> - main - ERROR - Main Error</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">55</span>:<span class="number">56</span>,<span class="number">259</span> - main.core - INFO - Core Info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">16</span>:<span class="number">55</span>:<span class="number">56</span>,<span class="number">259</span> - main.core - ERROR - Core Error</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到父子模块都使用了同样的输出配置。 如此一来，我们只要在入口文件里面定义好 logging 模块的输出配置，子模块只需要在定义 Logger 对象时名称使用父模块的名称开头即可共享配置，非常方便。</p>
                  <h3 id="文件配置"><a href="#文件配置" class="headerlink" title="文件配置"></a>文件配置</h3>
                  <p>在开发过程中，将配置在代码里面写死并不是一个好的习惯，更好的做法是将配置写在配置文件里面，我们可以将配置写入到配置文件，然后运行时读取配置文件里面的配置，这样是更方便管理和维护的，下面我们以一个实例来说明一下，首先我们定义一个 yaml 配置文件：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">version</span>: <span class="number">1</span></span><br><span class="line"><span class="attribute">formatters</span>:</span><br><span class="line">  <span class="attribute">brief</span>:</span><br><span class="line">    <span class="attribute">format</span>: <span class="string">"%(asctime)s - %(message)s"</span></span><br><span class="line">  <span class="attribute">simple</span>:</span><br><span class="line">    <span class="attribute">format</span>: <span class="string">"%(asctime)s - %(name)s - %(levelname)s - %(message)s"</span></span><br><span class="line"><span class="attribute">handlers</span>:</span><br><span class="line">  <span class="attribute">console</span>:</span><br><span class="line">    <span class="attribute">class </span>: logging.StreamHandler</span><br><span class="line">    <span class="attribute">formatter</span>: brief</span><br><span class="line">    <span class="attribute">level   </span>: INFO</span><br><span class="line">    <span class="attribute">stream  </span>: <span class="attribute">ext</span>:<span class="comment">//sys.stdout</span></span><br><span class="line">  <span class="attribute">file</span>:</span><br><span class="line">    <span class="attribute">class </span>: logging.FileHandler</span><br><span class="line">    <span class="attribute">formatter</span>: simple</span><br><span class="line">    <span class="attribute">level</span>: DEBUG</span><br><span class="line">    <span class="attribute">filename</span>: debug.log</span><br><span class="line">  <span class="attribute">error</span>:</span><br><span class="line">    <span class="attribute">class</span>: logging.handlers.RotatingFileHandler</span><br><span class="line">    <span class="attribute">level</span>: ERROR</span><br><span class="line">    <span class="attribute">formatter</span>: simple</span><br><span class="line">    <span class="attribute">filename</span>: error.log</span><br><span class="line">    <span class="attribute">maxBytes</span>: <span class="number">10485760</span></span><br><span class="line">    <span class="attribute">backupCount</span>: <span class="number">20</span></span><br><span class="line">    <span class="attribute">encoding</span>: utf8</span><br><span class="line"><span class="attribute">loggers</span>:</span><br><span class="line">  main.<span class="attribute">core</span>:</span><br><span class="line">    <span class="attribute">level</span>: DEBUG</span><br><span class="line">    <span class="attribute">handlers</span>: [console, file, error]</span><br><span class="line"><span class="attribute">root</span>:</span><br><span class="line">  <span class="attribute">level</span>: DEBUG</span><br><span class="line">  <span class="attribute">handlers</span>: [console]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了 formatters、handlers、loggers、root 等模块，实际上对应的就是各个 Formatter、Handler、Logger 的配置，参数和它们的构造方法都是相同的。 接下来我们定义一个主入口文件，main.py，内容如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line">import core</span><br><span class="line">import yaml</span><br><span class="line">import logging.config</span><br><span class="line">import os</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">def setup_logging(<span class="attribute">default_path</span>=<span class="string">'config.yaml'</span>, <span class="attribute">default_level</span>=logging.INFO):</span><br><span class="line">    path = default_path</span><br><span class="line">    <span class="keyword">if</span> os.path.exists(path):</span><br><span class="line">        with open(path, <span class="string">'r'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>) as f:</span><br><span class="line">           <span class="built_in"> config </span>= yaml.load(f)</span><br><span class="line">            logging.config.dictConfig(config)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        logging.basicConfig(<span class="attribute">level</span>=default_level)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">def log():</span><br><span class="line">    logging.<span class="builtin-name">debug</span>(<span class="string">'Start'</span>)</span><br><span class="line">    logging.<span class="builtin-name">info</span>(<span class="string">'Exec'</span>)</span><br><span class="line">    logging.<span class="builtin-name">info</span>(<span class="string">'Finished'</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    yaml_path = <span class="string">'config.yaml'</span></span><br><span class="line">    setup_logging(yaml_path)</span><br><span class="line">    log()</span><br><span class="line">    core.<span class="builtin-name">run</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了一个 setup_logging() 方法，里面读取了 yaml 文件的配置，然后通过 dictConfig() 方法将配置项传给了 logging 模块进行全局初始化。 另外这个模块还引入了另外一个模块 core，所以我们定义 core.py 如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logger = logging.getLogger(<span class="string">'main.core'</span>)</span><br><span class="line"></span><br><span class="line">def <span class="builtin-name">run</span>():</span><br><span class="line">    logger.<span class="builtin-name">info</span>(<span class="string">'Core Info'</span>)</span><br><span class="line">    logger.<span class="builtin-name">debug</span>(<span class="string">'Core Debug'</span>)</span><br><span class="line">    logger.<span class="builtin-name">error</span>(<span class="string">'Core Error'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个文件的内容和上文是没有什么变化的。 观察配置文件，主入口文件 main.py 实际上对应的是 root 一项配置，它指定了 handlers 是 console，即只输出到控制台。另外在 loggers 一项配置里面，我们定义了 main.core 模块，handlers 是 console、file、error 三项，即输出到控制台、输出到普通文件和回滚文件。 这样运行之后，我们便可以看到所有的运行结果输出到了控制台：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - Exec</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - Finished</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - Core Info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - Core Info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">728</span> - Core Error</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">728</span> - Core Error</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 debug.log 文件中则包含了 core.py 的运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - main.core - INFO - Core Info</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">727</span> - main.core - DEBUG - Core Debug</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">17</span>:<span class="number">07</span>:<span class="number">12</span>,<span class="number">728</span> - main.core - ERROR - Core Error</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，通过配置文件，我们可以非常灵活地定义 Handler、Formatter、Logger 等配置，同时也显得非常直观，也非常容易维护，在实际项目中，推荐使用此种方式进行配置。 以上便是 logging 模块的基本使用方法，有了它，我们可以方便地进行日志管理和维护，会给我们的工作带来极大的方便。</p>
                  <h2 id="日志记录使用常见误区"><a href="#日志记录使用常见误区" class="headerlink" title="日志记录使用常见误区"></a>日志记录使用常见误区</h2>
                  <p>在日志输出的时候经常我们会用到字符串拼接的形式，很多情况下我们可能会使用字符串的 format() 来构造一个字符串，但这其实并不是一个好的方法，因为还有更好的方法，下面我们对比两个例子：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logging.basicConfig(<span class="attribute">level</span>=logging.DEBUG, <span class="attribute">format</span>=<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># bad</span></span><br><span class="line">logging.<span class="builtin-name">debug</span>(<span class="string">'Hello &#123;0&#125;, &#123;1&#125;!'</span>.format(<span class="string">'World'</span>, <span class="string">'Congratulations'</span>))</span><br><span class="line"><span class="comment"># good</span></span><br><span class="line">logging.<span class="builtin-name">debug</span>(<span class="string">'Hello %s, %s!'</span>, <span class="string">'World'</span>, <span class="string">'Congratulations'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里有两种打印 Log 的方法，第一种使用了字符串的 format() 的方法进行构造，传给 logging 的只用到了第一个参数，实际上 logging 模块提供了字符串格式化的方法，我们只需要在第一个参数写上要打印输出的模板，占位符用 %s、%d 等表示即可，然后在后续参数添加对应的值就可以了，推荐使用这种方法。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">22</span>:<span class="number">27</span>:<span class="number">51</span>,<span class="number">220</span> - root - DEBUG - Hello World, Congratulations!</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">22</span>:<span class="number">27</span>:<span class="number">51</span>,<span class="number">220</span> - root - DEBUG - Hello World, Congratulations!</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://user-gold-cdn.xitu.io/2018/6/3/163c618ce73174a9?imageView2/0/w/1280/h/960/format/webp/ignore-error/1" alt=""></p>
                  <p>另外在进行异常处理的时候，通常我们会直接将异常进行字符串格式化，但其实可以直接指定一个参数将 traceback 打印出来，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line"></span><br><span class="line">logging.basicConfig(<span class="attribute">level</span>=logging.DEBUG, <span class="attribute">format</span>=<span class="string">'%(asctime)s - %(name)s - %(levelname)s - %(message)s'</span>)</span><br><span class="line"></span><br><span class="line">try:</span><br><span class="line">    result = 5 / 0</span><br><span class="line">except Exception as e:</span><br><span class="line">    # bad</span><br><span class="line">    logging.<span class="builtin-name">error</span>(<span class="string">'Error: %s'</span>, e)</span><br><span class="line">    # good</span><br><span class="line">    logging.<span class="builtin-name">error</span>(<span class="string">'Error'</span>, <span class="attribute">exc_info</span>=<span class="literal">True</span>)</span><br><span class="line">    # good</span><br><span class="line">    logging.exception(<span class="string">'Error'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果我们直接使用字符串格式化的方法将错误输出的话，是不会包含 Traceback 信息的，但如果我们加上 exc_info 参数或者直接使用 exception() 方法打印的话，那就会输出 Traceback 信息了。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">22</span>:<span class="number">24</span>:<span class="number">31</span>,<span class="number">927</span> - root - ERROR - Error: division by zero</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">22</span>:<span class="number">24</span>:<span class="number">31</span>,<span class="number">927</span> - root - ERROR - Error</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File <span class="string">"/private/var/books/aicodes/loggingtest/demo9.py"</span>, line <span class="number">6</span>, <span class="keyword">in</span> &lt;module&gt;</span><br><span class="line">    result = <span class="number">5</span> / <span class="number">0</span></span><br><span class="line">ZeroDivisionError: division by zero</span><br><span class="line"><span class="number">2018</span><span class="number">-06</span><span class="number">-03</span> <span class="number">22</span>:<span class="number">24</span>:<span class="number">31</span>,<span class="number">928</span> - root - ERROR - Error</span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File <span class="string">"/private/var/books/aicodes/loggingtest/demo9.py"</span>, line <span class="number">6</span>, <span class="keyword">in</span> &lt;module&gt;</span><br><span class="line">    result = <span class="number">5</span> / <span class="number">0</span></span><br><span class="line">ZeroDivisionError: division by zero</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://user-gold-cdn.xitu.io/2018/6/3/163c618f69092989?imageView2/0/w/1280/h/960/format/webp/ignore-error/1" alt=""></p>
                  <p>以上便是整个对 logging 模块的介绍。嗯，是时候抛弃 print 了，开始体验下 logging 的便利吧！</p>
                  <h2 id="参考内容"><a href="#参考内容" class="headerlink" title="参考内容"></a>参考内容</h2>
                  <ul>
                    <li><a href="https://docs.python.org/3/library/logging.html" target="_blank" rel="noopener">https://docs.python.org/3/library/logging.html</a> * <a href="http://www.cnblogs.com/dahu-daqing/p/7040764.html" target="_blank" rel="noopener">http://www.cnblogs.com/dahu-daqing/p/7040764.html</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-06-03 23:38:38" itemprop="dateCreated datePublished" datetime="2018-06-03T23:38:38+08:00">2018-06-03</time>
                </span>
                <span id="/6080.html" class="post-meta-item leancloud_visitors" data-flag-title="Python中logging模块的基本用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>16k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>14 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/6009.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/6009.html" class="post-title-link" itemprop="url">《Python3网络爬虫开发实战》来了！</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>嗨~ 给大家重磅推荐一本新书！还未上市前就已经重印 3 次的 Python 爬虫书！那么它就是由静觅博客博主崔庆才所作的《Python3网络爬虫开发实战》！！！ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/05/Python-3网格爬虫开发实战-立体图-857x1100-857x1100.jpg" alt=""></p>
                  <h2 id="书籍介绍"><a href="#书籍介绍" class="headerlink" title="书籍介绍"></a><strong>书籍介绍</strong></h2>
                  <p>本书<strong>《Python3网络爬虫开发实战》</strong>全面介绍了利用 Python3 开发网络爬虫的知识，书中首先详细介绍了各种类型的环境配置过程和爬虫基础知识，还讨论了 urllib、requests 等请求库和 Beautiful Soup、XPath、pyquery 等解析库以及文本和各类数据库的存储方法，另外本书通过多个真实新鲜案例介绍了分析 Ajax 进行数据爬取，Selenium 和 Splash 进行动态网站爬取的过程，接着又分享了一些切实可行的爬虫技巧，比如使用代理爬取和维护动态代理池的方法、ADSL 拨号代理的使用、各类验证码（图形、极验、点触、宫格等）的破解方法、模拟登录网站爬取的方法及 Cookies 池的维护等等。 此外，本书的内容还远远不止这些，作者还结合移动互联网的特点探讨了使用 Charles、mitmdump、Appium 等多种工具实现 App 抓包分析、加密参数接口爬取、微信朋友圈爬取的方法。此外本书还详细介绍了 pyspider 框架、Scrapy 框架的使用和分布式爬虫的知识，另外对于优化及部署工作，本书还包括 Bloom Filter 效率优化、Docker 和 Scrapyd 爬虫部署、分布式爬虫管理框架Gerapy 的分享。 全书共 604 页，足足两斤重呢~ 定价为 99 元！</p>
                  <h2 id="作者介绍"><a href="#作者介绍" class="headerlink" title="作者介绍"></a><strong>作者介绍</strong></h2>
                  <p>看书就先看看谁写的嘛，我们来了解一下~ 崔庆才，静觅博客博主（<a href="https://cuiqingcai.com），博客" target="_blank" rel="noopener">https://cuiqingcai.com），博客</a> Python 爬虫博文已过百万，北京航空航天大学硕士，微软小冰大数据工程师，有多个大型分布式爬虫项目经验，乐于技术分享，文章通俗易懂 ^<em>^ 附皂片一张 ~(@^</em>^@)~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG785-825x1100.jpeg" alt=""></p>
                  <h2 id="图文介绍"><a href="#图文介绍" class="headerlink" title="图文介绍"></a><strong>图文介绍</strong></h2>
                  <p>呕心沥血设计的宣传图也得放一下~ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/WechatIMG556.jpeg" alt=""></p>
                  <h2 id="专家评论"><a href="#专家评论" class="headerlink" title="专家评论"></a><strong>专家评论</strong></h2>
                  <p>书是好是坏，得让专家看评一评呀，那么下面就是几位专家的精彩评论，快来看看吧~ 在互联网软件开发工程师的分类中，爬虫工程师是非常重要的。爬虫工作往往是一个公司核心业务开展的基础，数据抓取下来，才有后续的加工处理和最终展现。此时数据的抓取规模、稳定性、实时性、准确性就显得非常重要。早期的互联网充分开放互联，数据获取的难度很小。随着各大公司对数据资产日益看重，反爬水平也在不断提高，各种新技术不断给爬虫软件提出新的课题。本书作者对爬虫的各个领域都有深刻研究，书中探讨了Ajax数据的抓取、动态渲染页面的抓取、验证码识别、模拟登录等高级话题，同时也结合移动互联网的特点探讨了App的抓取等。更重要的是，本书提供了大量源码，可以帮助读者更好地理解相关内容。强烈推荐给各位技术爱好者阅读！</p>
                  <p><strong>——梁斌</strong>，八友科技总经理</p>
                  <p>数据既是当今大数据分析的前提，也是各种人工智能应用场景的基础。得数据者得天下，会爬虫者走遍天下也不怕！一册在手，让小白到老司机都能有所收获！</p>
                  <p><strong>——李舟军</strong>，北京航空航天大学教授，博士生导师</p>
                  <p>本书从爬虫入门到分布式抓取，详细介绍了爬虫技术的各个要点，并针对不同的场景提出了对应的解决方案。另外，书中通过大量的实例来帮助读者更好地学习爬虫技术，通俗易懂，干货满满。强烈推荐给大家！</p>
                  <p><strong>——宋睿华</strong>，微软小冰首席科学家</p>
                  <p>有人说中国互联网的带宽全给各种爬虫占据了，这说明网络爬虫的重要性以及中国互联网数据封闭垄断的现状。爬是一种能力，爬是为了不爬。</p>
                  <p><strong>——施水才</strong>，北京拓尔思信息技术股份有限公司总裁</p>
                  <h2 id="全书目录"><a href="#全书目录" class="headerlink" title="全书目录"></a><strong>全书目录</strong></h2>
                  <p>书的目录也有~ 看这里！</p>
                  <ul>
                    <li><strong>1-开发环境配置</strong></li>
                    <li>1.1-Python3的安装</li>
                    <li>1.2-请求库的安装</li>
                    <li>1.3-解析库的安装</li>
                    <li>1.4-数据库的安装</li>
                    <li>1.5-存储库的安装</li>
                    <li>1.6-Web库的安装</li>
                    <li>1.7-App爬取相关库的安装</li>
                    <li>1.8-爬虫框架的安装</li>
                    <li>1.9-部署相关库的安装</li>
                    <li><strong>2-爬虫基础</strong></li>
                    <li>2.1-HTTP基本原理</li>
                    <li>2.2-网页基础</li>
                    <li>2.3-爬虫的基本原理</li>
                    <li>2.4-会话和Cookies</li>
                    <li>2.5-代理的基本原理</li>
                    <li><strong>3-基本库的使用</strong></li>
                    <li>3.1-使用urllib</li>
                    <li>3.1.1-发送请求</li>
                    <li>3.1.2-处理异常</li>
                    <li>3.1.3-解析链接</li>
                    <li>3.1.4-分析Robots协议</li>
                    <li>3.2-使用requests</li>
                    <li>3.2.1-基本用法</li>
                    <li>3.2.2-高级用法</li>
                    <li>3.3-正则表达式</li>
                    <li>3.4-抓取猫眼电影排行</li>
                    <li><strong>4-解析库的使用</strong></li>
                    <li>4.1-使用XPath</li>
                    <li>4.2-使用Beautiful Soup</li>
                    <li>4.3-使用pyquery</li>
                    <li><strong>5-数据存储</strong></li>
                    <li>5.1-文件存储</li>
                    <li>5.1.1-TXT文本存储</li>
                    <li>5.1.2-JSON文件存储</li>
                    <li>5.1.3-CSV文件存储</li>
                    <li>5.2-关系型数据库存储</li>
                    <li>5.2.1-MySQL存储</li>
                    <li>5.3-非关系型数据库存储</li>
                    <li>5.3.1-MongoDB存储</li>
                    <li>5.3.2-Redis存储</li>
                    <li><strong>6-Ajax数据爬取</strong></li>
                    <li>6.1-什么是Ajax</li>
                    <li>6.2-Ajax分析方法</li>
                    <li>6.3-Ajax结果提取</li>
                    <li>6.4-分析Ajax爬取今日头条街拍美图</li>
                    <li><strong>7-动态渲染页面爬取</strong></li>
                    <li>7.1-Selenium的使用</li>
                    <li>7.2-Splash的使用</li>
                    <li>7.3-Splash负载均衡配置</li>
                    <li>7.4-使用Selenium爬取淘宝商品</li>
                    <li><strong>8-验证码的识别</strong></li>
                    <li>8.1-图形验证码的识别</li>
                    <li>8.2-极验滑动验证码的识别</li>
                    <li>8.3-点触验证码的识别</li>
                    <li>8.4-微博宫格验证码的识别</li>
                    <li><strong>9-代理的使用</strong></li>
                    <li>9.1-代理的设置</li>
                    <li>9.2-代理池的维护</li>
                    <li>9.3-付费代理的使用</li>
                    <li>9.4-ADSL拨号代理</li>
                    <li>9.5-使用代理爬取微信公众号文章</li>
                    <li><strong>10-模拟登录</strong></li>
                    <li>10.1-模拟登录并爬取GitHub</li>
                    <li>10.2-Cookies池的搭建</li>
                    <li><strong>11-App的爬取</strong></li>
                    <li>11.1-Charles的使用</li>
                    <li>11.2-mitmproxy的使用</li>
                    <li>11.3-mitmdump爬取“得到”App电子书信息</li>
                    <li>11.4-Appium的基本使用</li>
                    <li>11.5-Appium爬取微信朋友圈</li>
                    <li>11.6-Appium+mitmdump爬取京东商品</li>
                    <li><strong>12-pyspider框架的使用</strong></li>
                    <li>12.1-pyspider框架介绍</li>
                    <li>12.2-pyspider的基本使用</li>
                    <li>12.3-pyspider用法详解</li>
                    <li><strong>13-Scrapy框架的使用</strong></li>
                    <li>13.1-Scrapy框架介绍</li>
                    <li>13.2-Scrapy入门</li>
                    <li>13.3-Selector的用法</li>
                    <li>13.4-Spider的用法</li>
                    <li>13.5-Downloader Middleware的用法</li>
                    <li>13.6-Spider Middleware的用法</li>
                    <li>13.7-Item Pipeline的用法</li>
                    <li>13.8-Scrapy对接Selenium</li>
                    <li>13.9-Scrapy对接Splash</li>
                    <li>13.10-Scrapy通用爬虫</li>
                    <li>13.11-Scrapyrt的使用</li>
                    <li>13.12-Scrapy对接Docker</li>
                    <li>13.13-Scrapy爬取新浪微博</li>
                    <li><strong>14-分布式爬虫</strong></li>
                    <li>14.1-分布式爬虫原理</li>
                    <li>14.2-Scrapy-Redis源码解析</li>
                    <li>14.3-Scrapy分布式实现</li>
                    <li>14.4-Bloom Filter的对接</li>
                    <li><strong>15-分布式爬虫的部署</strong></li>
                    <li>15.1-Scrapyd分布式部署</li>
                    <li>15.2-Scrapyd-Client的使用</li>
                    <li>15.3-Scrapyd对接Docker</li>
                    <li>15.4-Scrapyd批量部署</li>
                    <li>15.5-Gerapy分布式管理</li>
                  </ul>
                  <h2 id="购买链接"><a href="#购买链接" class="headerlink" title="购买链接"></a><strong>购买链接</strong></h2>
                  <p>想必很多小伙伴已经等了很久了，之前预售那么久也一直迟迟没有货，发售就有不少网店又售空了，不过现在起不用担心了！</p>
                  <p>书籍现已在京东、天猫、当当等网店上架并全面供应啦，复制链接到浏览器打开或扫描二维码打开即可购买了！</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/京东二维码.png" alt=""></p>
                  <p> 京东商城</p>
                  <p><a href="https://item.jd.com/12333540.html" target="_blank" rel="noopener">https://item.jd.com/12333540.html</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/天猫二维码.png" alt=""></p>
                  <p> 天猫商城</p>
                  <p><a href="https://detail.tmall.com/item.htm?id=566699703917" target="_blank" rel="noopener">https://detail.tmall.com/item.htm?id=566699703917</a></p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/当当二维码.png" alt=""></p>
                  <p>当当网</p>
                  <p><a href="http://product.dangdang.com/25249602.html" target="_blank" rel="noopener">http://product.dangdang.com/25249602.html</a></p>
                  <p>欢迎大家购买，谢谢支持！O(∩_∩)O</p>
                  <h2 id="免费预览"><a href="#免费预览" class="headerlink" title="免费预览"></a><strong>免费预览</strong></h2>
                  <p>不放心？想先看看有些啥，没问题！看这里： 免费章节试读（复制粘贴至浏览器打开）： <a href="https://cuiqingcai.com/5052.html">https://cuiqingcai.com/5052.html</a> 将一直免费开放<strong>前7章节</strong>，欢迎大家试读！</p>
                  <h2 id="视频教程"><a href="#视频教程" class="headerlink" title="视频教程"></a>视频教程</h2>
                  <p>当然除了书籍，也有配套的视频课程，作者同样是崔庆才，二者结合学习效果更佳！限时优惠折扣中！扫描下图中二维码即可了解详情！ <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/06/%E8%AF%BE%E7%A8%8B%E5%AE%A3%E4%BC%A0%E5%9B%BE.png" alt=""> 视频教程链接： <a href="https://edu.hellobi.com/course/157" target="_blank" rel="noopener">https://edu.hellobi.com/course/157</a> <a href="http://study.163.com/course/courseMain.htm?courseId=1003827039" target="_blank" rel="noopener">http://study.163.com/course/courseMain.htm?courseId=1003827039</a> 感谢大家支持！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-05-04 21:36:24" itemprop="dateCreated datePublished" datetime="2018-05-04T21:36:24+08:00">2018-05-04</time>
                </span>
                <span id="/6009.html" class="post-meta-item leancloud_visitors" data-flag-title="《Python3网络爬虫开发实战》来了！" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5984.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5984.html" class="post-title-link" itemprop="url">Flask 静态文件缓存问题</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="大家好，今天才发现很多学习Flask的小伙伴都有这么一个问题，清理缓存好麻烦啊，今天就教大家怎么解决。"><a href="#大家好，今天才发现很多学习Flask的小伙伴都有这么一个问题，清理缓存好麻烦啊，今天就教大家怎么解决。" class="headerlink" title="大家好，今天才发现很多学习Flask的小伙伴都有这么一个问题，清理缓存好麻烦啊，今天就教大家怎么解决。"></a>大家好，今天才发现很多学习Flask的小伙伴都有这么一个问题，清理缓存好麻烦啊，今天就教大家怎么解决。</h1>
                  <p>大家在使用Flask静态文件的时候，每次更新，发现CSS或是Js或者其他的文件不会更新。 这是因为浏览器的缓存问题。 普遍大家是这几步解决办法。</p>
                  <ul>
                    <li><strong>清理浏览器缓存</strong></li>
                    <li><strong>设置浏览器不缓存</strong></li>
                    <li>也有以下这么写的</li>
                  </ul>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">@app.context_processor</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">override_url_for</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="keyword">return</span> dict(url_for=dated_url_for)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">dated_url_for</span><span class="params">(endpoint, **values)</span>:</span></span><br><span class="line">    <span class="keyword">if</span> endpoint == <span class="string">'static'</span>:</span><br><span class="line">        filename = values.get(<span class="string">'filename'</span>, <span class="literal">None</span>)</span><br><span class="line">    <span class="keyword">if</span> filename:</span><br><span class="line">        file_path = os.path.join(app.root_path, endpoint, filename)</span><br><span class="line">        values[<span class="string">'q'</span>] = int(os.stat(file_path).st_mtime)</span><br><span class="line">        <span class="keyword">return</span> url_for(endpoint, **values)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果是我，我不会这么做，效率很低。 <img src="https://note.youdao.com/yws/api/personal/file/WEB509e95a33af9869430a40426c0ebf665?method=download&amp;shareKey=be706f1bce4b10e5d615e902e712b2d0" alt=""> 这是 Flask的 <strong><code>config</code></strong> 的源码，里面可以看到，有设置缓存最大时间 <strong><code>SEND_FILE_MAX_AGE_DEFAULT</code></strong> 可以看到，它是一个 <strong>temedelta</strong> 的值 <strong>我们去更改配置。</strong> <img src="https://note.youdao.com/yws/api/personal/file/WEB2929bc40de12a0530fcad5b9bdd8e2bc?method=download&amp;shareKey=72db34d3085f531add452f0ae8517041" alt=""> 第2行: <strong>我们引入了<code>datetime</code>的<code>timedelta</code>对象</strong> 第6行: <strong>我们配置缓存最大时间</strong> <strong>这样就解决了缓存问题，不用去写多余的代码，不用去清理浏览器的缓存。</strong> <strong>一定要学着去看官方文档和框架的源代码！！</strong> </p>
                  <p><strong>有什么问题请联系</strong></p>
                  <p>[caption id=”attachment_5953” align=”aligncenter” width=”320”]<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/04/WechatIMG1-1-320x320.jpeg" alt=""> 微信二维码[/caption]</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/蒋翔宇" class="author" itemprop="url" rel="index">蒋翔宇</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-04-15 12:19:10" itemprop="dateCreated datePublished" datetime="2018-04-15T12:19:10+08:00">2018-04-15</time>
                </span>
                <span id="/5984.html" class="post-meta-item leancloud_visitors" data-flag-title="Flask 静态文件缓存问题" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>751</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5876.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Linux <i class="label-arrow"></i>
                  </a>
                  <a href="/5876.html" class="post-title-link" itemprop="url">SSH反向隧道搭建过程</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>现在我们有一台内网主机 A，在局域网内是可以访问的，但是如果我们现在不处在局域网内，可以选择 VPN 连接，但这样其实并不太方便，所以本节我们来说明一下利用 SSH 反向隧道来实现访问内网主机的方法。</p>
                  <h2 id="准备"><a href="#准备" class="headerlink" title="准备"></a>准备</h2>
                  <p>首先我们需要有一台公网主机作为跳板，这台主机是可以公网访问的，我们将其命名为 B，它的 IP 假设为 10.10.10.10。 所以两台机器网络配置如下：</p>
                  <h3 id="A-内网机器"><a href="#A-内网机器" class="headerlink" title="A 内网机器"></a>A 内网机器</h3>
                  <ul>
                    <li>IP：192.168.1.2</li>
                    <li>SSH端口： 22</li>
                    <li>用户名：usera</li>
                    <li>密码：passworda</li>
                    <li>内网配置端口：22（即配置 SSH 端口的反向隧道）</li>
                  </ul>
                  <h3 id="B-公网机器"><a href="#B-公网机器" class="headerlink" title="B 公网机器"></a>B 公网机器</h3>
                  <ul>
                    <li>IP：10.10.10.10</li>
                    <li>SSH端口： 22</li>
                    <li>用户名：userb</li>
                    <li>密码：passwordb</li>
                    <li>公网端口：22001（即用 B 的 22001 端口连到 A 的 SSH 22 端口）</li>
                  </ul>
                  <h2 id="配置SSH秘钥"><a href="#配置SSH秘钥" class="headerlink" title="配置SSH秘钥"></a>配置SSH秘钥</h2>
                  <p>首先我们需要在 A 主机上生成 SSH 秘钥，和 B 用 SSH 建立认证。 首先在主机 A 上执行如下命令生成 SSH 秘钥：</p>
                  <figure class="highlight excel">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh-keygen -<span class="built_in">t</span> rsa -C <span class="string">"your@email.com"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令里面的邮箱需要自行更换。 然后利用如下命令将 A 的 SSH 秘钥添加到 B 的 authorized_keys 里面：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh-copy-id <span class="symbol">userb@</span><span class="number">10.10</span><span class="number">.10</span><span class="number">.10</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行后会提示输入主机 B 的密码，执行完毕之后，我们登录到 B，就发现 authorized_keys 里面就多了 A 的 SSH 公钥了，成功建立 SSH 认证。</p>
                  <h2 id="B-主机配置"><a href="#B-主机配置" class="headerlink" title="B 主机配置"></a>B 主机配置</h2>
                  <p>B 主机需要更改 /etc/ssh/sshd_config 文件，修改如下一行：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">GatewayPorts</span> <span class="literal">yes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样可以把监听的端口绑定到任意IP 0.0.0.0上，否则只有本机 127.0.0.1 可以访问。 然后重启 sshd 服务：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo<span class="built_in"> service </span>sshd restart</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="A-主机配置"><a href="#A-主机配置" class="headerlink" title="A 主机配置"></a>A 主机配置</h2>
                  <p>主机 A 再安装一个 AutoSSH，以 Ubuntu 为例，命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> install autossh</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后执行如下命令即可完成反向 SSH 配置：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">auto</span>ssh -M <span class="number">55555</span> -NfR <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span>:<span class="number">22001</span>:localhost:<span class="number">22</span> <span class="symbol">userb@</span><span class="number">10.10</span><span class="number">.10</span><span class="number">.10</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 -M 后面任意填写一个可用端口即可，-N 代表只建立连接，不打开shell ，-f 代表建立成功后在后台运行，-R 代表指定端口映射。 这里是将 A 主机的 22 端口映射到 B 主机的 22001 端口，这样就完成了配置。 主要我们再访问 B 主机的 22001 端口，就会自动转发到 A 主机的 22 端口了，即可以公网访问了。</p>
                  <h2 id="连接测试"><a href="#连接测试" class="headerlink" title="连接测试"></a>连接测试</h2>
                  <p>接下来 SSH 测试连接 A 主机即可：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh <span class="symbol">usera@</span><span class="number">10.10</span><span class="number">.10</span><span class="number">.10</span> -p <span class="number">22001</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输入密码，完成连接。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-26 02:39:00" itemprop="dateCreated datePublished" datetime="2018-03-26T02:39:00+08:00">2018-03-26</time>
                </span>
                <span id="/5876.html" class="post-meta-item leancloud_visitors" data-flag-title="SSH反向隧道搭建过程" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5873.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5873.html" class="post-title-link" itemprop="url">Attention原理及TensorFlow AttentionWrapper源码解析</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节来详细说明一下 Seq2Seq 模型中一个非常有用的 Attention 的机制，并结合 TensorFlow 中的 AttentionWrapper 来剖析一下其代码实现。</p>
                  <h2 id="Seq2Seq"><a href="#Seq2Seq" class="headerlink" title="Seq2Seq"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#seq2seq" target="_blank" rel="noopener"></a>Seq2Seq</h2>
                  <p>首先来简单说明一下 Seq2Seq 模型，如果搞过深度学习，想必一定听说过 Seq2Seq 模型，Seq2Seq 其实就是 Sequence to Sequence，也简称 S2S，也可以称之为 Encoder-Decoder 模型，这个模型的核心就是编码器（Encoder）和解码器（Decoder）组成的，架构雏形是在 2014 年由论文 <a href="https://arxiv.org/abs/1406.1078" target="_blank" rel="noopener">Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation, Cho et al</a> 提出的，后来 <a href="https://arxiv.org/abs/1409.3215" target="_blank" rel="noopener">Sequence to Sequence Learning with Neural Networks, Sutskever et al</a> 算是比较正式地提出了 Sequence to Sequence 的架构，后来 <a href="https://arxiv.org/abs/1409.0473" target="_blank" rel="noopener">Neural Machine Translation by Jointly Learning to Align and Translate, Bahdanau et al</a> 又提出了 Attention 机制，将 Seq2Seq 模型推上神坛，并横扫了非常多的任务，现在也非常广泛地用于机器翻译、对话生成、文本摘要生成等各种任务上，并取得了非常好的效果。 下面的图示意了 Seq2Seq 模型的基本架构： <a href="https://github.com/Germey/AI/blob/master/assets/2018-03-23-16-34-19.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-03-23-16-34-19.jpg" alt=""></a> 可以看到图中有一个中间状态 $ c $ 向量，在 $ c $ 向量左侧的我们可以称之为编码器（Encoder），编码器这里示意的是 RNN 序列，另外 RNN 单元还可以使用 LSTM、GRU 等变体， 在编码器下方输入了 $ x_1 $、$ x_2 $、$ x_3 $、$ x_4 $，代表模型的输入内容，例如在翻译模型中可以分别代表“我爱中国”这四个字，这样经过序列处理，它就会得到最后的输出，我们将其表示为 $ c $ 向量，这样编码器的工作就完成了。在图中 $ c $ 向量的右侧部分我们可以称之为解码器（Decoder），它拿到编码器生成的 $ c $ 向量，然后再进行序列解码，得到输出结果 $ y_1 $、$ y_2 $、$ y_3 $，例如刚才输入的“我爱中国”四个字便被解码成了 “I love China”，这样就实现了翻译任务，以上就是最基本的 Seq2Seq 模型原理。 另外还有一种变体，$ c $ 向量在每次解码的时候都会作为解码器的输入，其实原理都是类似的，如图所示： <a href="https://github.com/Germey/AI/blob/master/assets/2018-03-23-19-42-16.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-03-23-19-42-16.jpg" alt=""></a> 这种模型架构是通用的，所以它的适用场景也非常广泛。如机器翻译、对话生成、文本摘要、阅读理解、语音识别，也可以用在一些趣味场景中，如诗词生成、对联生成、代码生成、评论生成等等，效果都很不错。</p>
                  <h2 id="Attention"><a href="#Attention" class="headerlink" title="Attention"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#attention" target="_blank" rel="noopener"></a>Attention</h2>
                  <p>通过上图我们可以发现，Encoder 把所有的输入序列编码成了一个 $ c $ 向量，然后使用 $ c $ 向量来进行解码，因此，$ c $ 向量中必须包含了原始序列中的所有信息，所以它的压力其实是很大的，而且由于 RNN 容易把前面的信息“忘记”掉，所以基本的 Seq2Seq 模型，对于较短的输入来说，效果还是可以接受的，但是在输入序列比较长的时候，$ c $ 向量存不下那么多信息，就会导致生成效果大大折扣。 Attention 机制解决了这个问题，它可以使得在输入文本长的时候精确率也不会有明显下降，它是怎么做的呢？既然一个 $ c $ 向量存不了，那么就引入多个 $ c $ 向量，称之为 $ c<em>1 $、$ c_2 $、…、$ c_i $，在解码的时候，这里的 $ i $ 对应着 Decoder 的解码位次，每次解码就利用对应的 $ c_i $ 向量来解码，如图所示： <a href="https://github.com/Germey/AI/blob/master/assets/2018-03-23-17-38-45.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-03-23-17-38-45.jpg" alt=""></a> 这里的每个 $ c_i $ 向量其实包含了当前所输出与输入序列各个部分重要性的相关的信息。不同的 $ c_i $ 向量里面包含的输入信息各部分的权重是不同的，先放一个示意图： <a href="https://github.com/Germey/AI/blob/master/assets/2018-03-23-19-35-28.jpg" target="_blank" rel="noopener"><img src="https://github.com/Germey/AI/raw/master/assets/2018-03-23-19-35-28.jpg" alt=""></a> 还是上面的例子，例如输入信息是“我爱中国”，输出的的理想结果应该是“I love China”，在解码的时候，应该首先需要解码出 “I” 这个字符，这时候会用到 $ c_1 $ 向量，而 $ c_1 $ 向量包含的信息中，“我”这个字的重要性更大，因此它便倾向解码输出 “I”，当解码第二个字的时候，会用到 $ c_2 $ 向量，而 $ c_2 $ 向量包含的信息中，“爱” 这个字的重要性更大，因此会解码输出 “love”，在解码第三个字的时候，会用到 $ c_3 $ 向量，而 $ c_3 $向量包含的信息中，”中国” 这两个字的权重都比较大，因此会解码输出 “China”。所以其实，Attention 注意力机制中的 $ c_i $ 向量记录了不同解码时刻应该更关注于哪部分输入数据，也实现了编码解码过程的对齐。经过实验发现，这种机制可以有效解决输入信息过长时导致信息解码效果不理想的问题，另外解码生成效果同时也有提升。 下面我们以 Bahdanau 提出的 Attention 为例来详细剖析一下 Attention 机制。 在没有引入 Attention 之前，Decoder 在某个时刻解码的时候实际上是依赖于三个部分的，首先我们知道 RNN 中，每次输出结果会依赖于隐层和输入，在 Seq2Seq 模型中，还需要依赖于 $ c $ 向量，所以这里我们设在 $ i $ 时刻，解码器解码的内容是 $ y_i $，上一次解码结果是 $ y</em>{i-1} $，隐层输出是 $ s<em>t $，所以它们满足这样的关系： $$ y_i = g(y</em>{i-1}, s<em>i, c) <script type="math/tex">同时 $ s_i $ 和 $ c $ 还满足这样的关系：</script> s_i = f(s</em>{i-1}, y<em>{i-1}, c) <script type="math/tex">即每次的隐层输出是上一个隐层和上一个输出结果和 $ c $ 向量共同计算得出的。 但是刚才说了，这样会带来一些问题，$ c $ 向量不足以包含输入内容的所有信息，尤其是在输入序列特别长的情况下，所以这里我们不再使用一个 $ c $ 向量，而是每一个解码过程对应一个 $ c_i $ 向量，所以公式改写如下：</script> y_i = g(y</em>{i-1}, s<em>i, c_i) <script type="math/tex">同时 $ s_i $ 的计算方式也变为如下公式：</script> s_i = f(s</em>{i-1}, y<em>{i-1}, c_i) $$ 所以，这里每次解码得出 $ y_i $ 时，都有与之对应的 $ c_i $ 向量。那么这个 $ c_i $ 向量又是怎么来的呢？实际上它是由编码器端每个时刻的隐含状态加权平均得到的，这里假设编码器端的的序列长度为 $ T_x $，序列位次用 $ j $ 来表示，编码器段每个时刻的隐含状态即为 $ h_1 $、$ h_2 $、…、$ h_j $、…、$ h</em>{T<em>x} $，对于解码器的第 $ i $ 时刻，对应的 $ c_i $ 表示如下： $$ c_i = \sum</em>{j=1}^{T<em>x} \alpha</em>{ij}h<em>j $$ 编码器输出的结果中，$ h_j $ 中包含了输入序列中的第 $ j $ 个词及前面的一些信息，如果是用了双向 RNN 的话，则包含的是第 $ j $ 个词即前后的一些词的信息，这里 $ \alpha</em>{ij} $ 代表了分配的权重，这代表在生成第 i 个结果的时候，对于输入信息的各个阶段的 $ hj $ 的注意力分配是不同的。 当 $ a<em>{ij} $ 的值越高，表示第 $ i $ 个输出在第 $ j $ 个输入上分配的注意力越多，这样就会导致在生成第 $ i $ 个输出的时候，受第 $ j $ 个输入的影响也就越大。 那么 $ a</em>{ij} $ 又是怎么得来的呢？其实它就又关系到第 $ i-1 $ 个输出隐藏状态 $ s<em>{i-1} $ 以及输入中的各个隐含状态 $ h_j $，公式表示如下： $$ \alpha</em>{ij} = \frac {exp(e<em>{ij})} {\sum</em>{k=1}^{T<em>x} exp(e</em>{ik})} <script type="math/tex">同时 $ e_{ij} $ 又表示为：</script> e<em>{ij} = a(s</em>{i-1}, h<em>j) = {v_a}^Ttanh(W_as</em>{i-1} + U<em>ah_j) $$ 这也就是说，这个权重就是 $ s</em>{i-1} $ 和 $ h<em>j $ 分别计算得到一个数值，然后再过一个 softmax 函数得到的，结果就是 $ \alpha</em>{ij} $。 因此 $ c<em>i $ 就可以表示为： $$ c_i = \sum</em>{j=1}^{T<em>x} softmax(a(s</em>{i-1}, h_j)) \cdot h_j $$ 以上便是整个 Attention 机制的推导过程。</p>
                  <h2 id="TensorFlow-AttentionWrapper"><a href="#TensorFlow-AttentionWrapper" class="headerlink" title="TensorFlow AttentionWrapper"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#tensorflow-attentionwrapper" target="_blank" rel="noopener"></a>TensorFlow AttentionWrapper</h2>
                  <p>我们了解了基本原理，但真正离程序实现出来其实还是有很大差距的，接下来我们就结合 TensorFlow 框架来了解一下 Attention 的实现机制。 在 TensorFlow 中，Attention 的相关实现代码是在 tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py 文件中，这里面实现了两种 Attention 机制，分别是 BahdanauAttention 和 LuongAttention，其实现论文分别如下：</p>
                  <ul>
                    <li><a href="https://arxiv.org/abs/1409.0473" target="_blank" rel="noopener">Neural Machine Translation by Jointly Learning to Align and Translate, Bahdanau, et al</a></li>
                    <li><a href="https://arxiv.org/abs/1508.04025" target="_blank" rel="noopener">Effective Approaches to Attention-based Neural Machine Translation, Luong, et al</a></li>
                  </ul>
                  <p>整个 attention_wrapper.py 文件中主要包含几个类，我们主要关注其中几个：</p>
                  <ul>
                    <li>AttentionMechanism、_BaseAttentionMechanism、LuongAttention、BahdanauAttention 实现了 Attention 机制的逻辑。<ul>
                        <li>AttentionMechanism 是 Attention 类的父类，继承了 object 类，内部没有任何实现。</li>
                        <li>_BaseAttentionMechanism 继承自 AttentionMechanism 类，定义了 Attention 机制的一些公共方法实现和属性。</li>
                        <li>LuongAttention、BahdanauAttention 均继承 _BaseAttentionMechanism 类，分别实现了上面两篇论文的 Attention 机制。</li>
                      </ul>
                    </li>
                    <li>AttentionWrapperState 用来存储整个计算过程中的 state，和 RNN 中的 state 类似，只不过这里额外还存储了 attention、time 等信息。</li>
                    <li>AttentionWrapper 主要用于对封装 RNNCell，继承自 RNNCell，封装后依然是 RNNCell 的实例，可以构建一个带有 Attention 机制的 Decoder。</li>
                    <li>另外还有一些公共方法，例如 hardmax、safe_cumpord 等。</li>
                  </ul>
                  <p>下面我们以 BahdanauAttention 为例来说明 Attention 机制及 AttentionWrapper 的实现。</p>
                  <h3 id="BahdanauAttention"><a href="#BahdanauAttention" class="headerlink" title="BahdanauAttention"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#bahdanauattention" target="_blank" rel="noopener"></a>BahdanauAttention</h3>
                  <p>首先我们来介绍 BahdanauAttention 类的具体原理。 首先我们来看下它的初始化方法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def __init__(self,</span><br><span class="line">    num_units,</span><br><span class="line">    memory,</span><br><span class="line">    <span class="attribute">memory_sequence_length</span>=None,</span><br><span class="line">    <span class="attribute">normalize</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">probability_fn</span>=None,</span><br><span class="line">    <span class="attribute">score_mask_value</span>=None,</span><br><span class="line">    <span class="attribute">dtype</span>=None,</span><br><span class="line">    <span class="attribute">name</span>=<span class="string">"BahdanauAttention"</span>):</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里一共接受八个参数，下面一一进行说明：</p>
                  <ul>
                    <li>num<em>units：神经元节点数，我们知道在计算 $ e</em>{ij} $ 的时候，需要使用 $ s_{i-1} $ 和 $ h_j $ 来进行计算，而二者的维度可能并不是统一的，需要进行变换和统一，所以这里就有了 $ W_a $ 和 $ U_a $ 这两个系数，所以在代码中就是用 num_units 来声明了一个全连接 Dense 网络，用于统一二者的维度，以便于下一步的计算：</li>
                  </ul>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">query_layer</span>=layers_core.Dense(num_units, <span class="attribute">name</span>=<span class="string">"query_layer"</span>, <span class="attribute">use_bias</span>=<span class="literal">False</span>, <span class="attribute">dtype</span>=dtype)</span><br><span class="line"><span class="attribute">memory_layer</span>=layers_core.Dense(num_units, <span class="attribute">name</span>=<span class="string">"memory_layer"</span>, <span class="attribute">use_bias</span>=<span class="literal">False</span>, <span class="attribute">dtype</span>=dtype)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们可以看到声明了一个 query<em>layer 和 memory_layer，分别和 $ s</em>{i-1} $ 及 $ h_j $ 做全连接变换，统一维度。</p>
                  <ul>
                    <li>memory：The memory to query; usually the output of an RNN encoder. 即解码时用到的上文信息，维度需要是 [batch_size, max_time, context_dim]。这时我们观察一下父类 _BaseAttentionMechanism 的初始化方法，实现如下：</li>
                  </ul>
                  <figure class="highlight gml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> ops.name_scope(</span><br><span class="line">    name, <span class="string">"BaseAttentionMechanismInit"</span>, nest.flatten(memory)):</span><br><span class="line">  <span class="literal">self</span>._values = _prepare_memory(</span><br><span class="line">      memory, memory_sequence_length,</span><br><span class="line">      check_inner_dims_defined=check_inner_dims_defined)</span><br><span class="line">  <span class="literal">self</span>._keys = (</span><br><span class="line">      <span class="literal">self</span>.memory_layer(<span class="literal">self</span>._values) <span class="keyword">if</span> <span class="literal">self</span>.memory_layer</span><br><span class="line">      <span class="keyword">else</span> <span class="literal">self</span>._values)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里通过 _prepare_memory() 方法对 memory 进行处理，然后调用 memory_layer 对 memory 进行全连接维度变换，变换成 [batch_size, max_time, num_units]。</p>
                  <ul>
                    <li>memory_sequence_length：Sequence lengths for the batch entries in memory. 即 memory 变量的长度信息，类似于 dynamic_rnn 中的 sequence_length，被 _prepare_memory() 方法调用处理 memory 变量，进行 mask 操作：</li>
                  </ul>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">seq_len_mask = <span class="built_in">array</span>_ops.sequence_mask(</span><br><span class="line">    memory_sequence_length,</span><br><span class="line">    maxlen=<span class="built_in">array</span>_ops.shape(nest.flatten(memory)[<span class="number">0</span>])[<span class="number">1</span>],</span><br><span class="line">    dtype=nest.flatten(memory)[<span class="number">0</span>].dtype)</span><br><span class="line">seq_len_batch_size = (</span><br><span class="line">    memory_sequence_length.shape[<span class="number">0</span>].value</span><br><span class="line">    <span class="keyword">or</span> <span class="built_in">array</span>_ops.shape(memory_sequence_length)[<span class="number">0</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>normalize：Whether to normalize the energy term. 即是否要实现标准化，方法出自论文：<a href="https://arxiv.org/abs/1602.07868" target="_blank" rel="noopener">Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks, Salimans, et al</a>。</li>
                    <li>probability_fn：A callable function which converts the score to probabilities. 计算概率时的函数，必须是一个可调用的函数，默认使用 softmax()，还可以指定 hardmax() 等函数。</li>
                    <li>score_mask_value：The mask value for score before passing into probability_fn. The default is -inf. Only used if memory_sequence_length is not None. 在使用 probability_fn 计算概率之前，对 score 预先进行 mask 使用的值，默认是负无穷。但这个只有在 memory_sequence_length 参数定义的时候有效。</li>
                    <li>dtype：The data type for the query and memory layers of the attention mechanism. 数据类型，默认是 float32。</li>
                    <li>name：Name to use when creating ops，自定义名称。</li>
                  </ul>
                  <p>接下来类里面定义了一个 <strong>call</strong>() 方法：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def <span class="constructor">__call__(<span class="params">self</span>, <span class="params">query</span>, <span class="params">previous_alignments</span>)</span>:</span><br><span class="line">    <span class="keyword">with</span> variable_scope.variable<span class="constructor">_scope(None, <span class="string">"bahdanau_attention"</span>, [<span class="params">query</span>])</span>:</span><br><span class="line">      processed_query = self.query<span class="constructor">_layer(<span class="params">query</span>)</span> <span class="keyword">if</span> self.query_layer <span class="keyword">else</span> query</span><br><span class="line">      score = <span class="constructor">_bahdanau_score(<span class="params">processed_query</span>, <span class="params">self</span>.<span class="params">_keys</span>, <span class="params">self</span>.<span class="params">_normalize</span>)</span></span><br><span class="line">    alignments = self.<span class="constructor">_probability_fn(<span class="params">score</span>, <span class="params">previous_alignments</span>)</span></span><br><span class="line">    return alignments</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先定义了 processed_query，这里也是通过 query_layer 过了一个全连接网络，将最后一维统一成 num_units，然后调用了 bahdanau_score() 方法，这个方法是比较重要的，主要用来计算公式中的 $ e{ij} $，传入的参数是 processed_query 以及上文中提及的 keys 变量，二者一个代表了 $ s{i-1} $，一个代表了 $ h_j $，_bahdanau_score() 方法实现如下：</p>
                  <figure class="highlight nix">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def _bahdanau_score(processed_query, keys, normalize):</span><br><span class="line">    <span class="attr">dtype</span> = processed_query.dtype</span><br><span class="line">    <span class="comment"># Get the number of hidden units from the trailing dimension of keys</span></span><br><span class="line">    <span class="attr">num_units</span> = keys.shape[<span class="number">2</span>].value <span class="literal">or</span> array_ops.shape(keys)[<span class="number">2</span>]</span><br><span class="line">    <span class="comment"># Reshape from [batch_size, ...] to [batch_size, 1, ...] for broadcasting.</span></span><br><span class="line">    <span class="attr">processed_query</span> = array_ops.expand_dims(processed_query, <span class="number">1</span>)</span><br><span class="line">    <span class="attr">v</span> = variable_scope.get_variable(</span><br><span class="line">      <span class="string">"attention_v"</span>, [num_units], <span class="attr">dtype=dtype)</span></span><br><span class="line">    <span class="keyword">if</span> normalize:</span><br><span class="line">        <span class="comment"># Scalar used in weight normalization</span></span><br><span class="line">        <span class="attr">g</span> = variable_scope.get_variable(</span><br><span class="line">            <span class="string">"attention_g"</span>, <span class="attr">dtype=dtype,</span></span><br><span class="line">            <span class="attr">initializer=math.sqrt((1.</span> / num_units)))</span><br><span class="line">        <span class="comment"># Bias added prior to the nonlinearity</span></span><br><span class="line">        <span class="attr">b</span> = variable_scope.get_variable(</span><br><span class="line">            <span class="string">"attention_b"</span>, [num_units], <span class="attr">dtype=dtype,</span></span><br><span class="line">            <span class="attr">initializer=init_ops.zeros_initializer())</span></span><br><span class="line">        <span class="comment"># normed_v = g * v / ||v||</span></span><br><span class="line">        <span class="attr">normed_v</span> = g * v * math_ops.rsqrt(</span><br><span class="line">            math_ops.reduce_sum(math_ops.square(v)))</span><br><span class="line">        return math_ops.reduce_sum(normed_v * math_ops.tanh(keys + processed_query + b), [<span class="number">2</span>])</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        return math_ops.reduce_sum(v * math_ops.tanh(keys + processed_query), [<span class="number">2</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里其实就是实现了 keys 和 processed<em>query 的加和，如果指定了 normalize 的话还需要进行额外的 normalize，结果就是公式中的 $ e</em>{ij} $，在 TensorFlow 中常用 score 变量表示。 接下来再回到 <strong>call</strong>() 方法中，这里得到了 score 变量，接下来可以对齐求 softmax() 操作，得到 $ \alpha_{ij} $：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">alignments = self.<span class="constructor">_probability_fn(<span class="params">score</span>, <span class="params">previous_alignments</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就代表了在 $ i $ 时刻，Decoder 的时候对 Encoder 得到的每个 $ hj $ 的权重大小比例，在 TensorFlow 中常用 alignments 变量表示。 所以综上所述，BahdanauAttention 就是初始化时传入 num_units 以及 Encoder Outputs，然后调时传入 query 用即可得到权重变量 alignments。</p>
                  <h3 id="AttentionWrapperState"><a href="#AttentionWrapperState" class="headerlink" title="AttentionWrapperState"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#attentionwrapperstate" target="_blank" rel="noopener"></a>AttentionWrapperState</h3>
                  <p>接下来我们再看下 AttentionWrapperState 这个类，这个类其实比较简单，就是定义了 Attention 过程中可能需要保存的变量，如 cell_state、attention、time、alignments 等内容，同时也便于后期的可视化呈现，代码实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">AttentionWrapperState</span><span class="params">(</span></span></span><br><span class="line"><span class="class"><span class="params">    collections.namedtuple<span class="params">(<span class="string">"AttentionWrapperState"</span>,</span></span></span></span><br><span class="line"><span class="class"><span class="params"><span class="params">                           <span class="params">(<span class="string">"cell_state"</span>, <span class="string">"attention"</span>, <span class="string">"time"</span>, <span class="string">"alignments"</span>,</span></span></span></span></span><br><span class="line"><span class="class"><span class="params"><span class="params"><span class="params">                            <span class="string">"alignment_history"</span>)</span>)</span>)</span>:</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见它就是继承了 namedtuple 这个数据结构，其实整个 AttentionWrapperState 就像声明了一个结构体，可以传入需要的字段生成这个对象。</p>
                  <h3 id="AttentionWrapper"><a href="#AttentionWrapper" class="headerlink" title="AttentionWrapper"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#attentionwrapper" target="_blank" rel="noopener"></a>AttentionWrapper</h3>
                  <p>了解了 Attention 机制及 BahdanauAttention 的原理之后，最后我们再来了解一下 AttentionWrapper，可能你用过很多其他的 Wrapper，如 DropoutWrapper、ResidualWrapper 等等，它们其实都是 RNNCell 的实例，其实 AttentionWrapper 也不例外，它对 RNNCell 进行了封装，封装后依然还是 RNNCell 的实例。一个普通的 RNN 模型，你要加入 Attention，只需要在 RNNCell 外面套一层 AttentionWrapper 并指定 AttentionMechanism 的实例就好了。而且如果要更换 AttentionMechanism，只需要改变 AttentionWrapper 的参数就好了，这可谓对 Attention 的实现架构完全解耦，配置非常灵活，TF 大法好！ 接下来我们首先来看下它的初始化方法，其参数是这样的：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def __init__(self,</span><br><span class="line">    cell,</span><br><span class="line">    attention_mechanism,</span><br><span class="line">    <span class="attribute">attention_layer_size</span>=None,</span><br><span class="line">    <span class="attribute">alignment_history</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">cell_input_fn</span>=None,</span><br><span class="line">    <span class="attribute">output_attention</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">initial_cell_state</span>=None,</span><br><span class="line">    <span class="attribute">name</span>=None):</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面对参数进行一一说明：</p>
                  <ul>
                    <li>cell：An instance of RNNCell. RNNCell 的实例，这里可以是单个的 RNNCell，也可以是多个 RNNCell 组成的 MultiRNNCell。</li>
                    <li>attention_mechanism：即 AttentionMechanism 的实例，如 BahdanauAttention 对象，另外可以是多个 AttentionMechanism 组成的列表。</li>
                    <li>attention_layer_size：是数字或者数字做成的列表，如果是 None（默认），直接使用加权计算后得到的 Attention 作为输出，如果不是 None，那么 Attention 结果还会和 Output 进行拼接并做线性变换再输出。其代码实现如下：</li>
                  </ul>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">if</span> attention_layer_size is not None:</span><br><span class="line">    attention_layer_sizes = tuple(attention_layer_size <span class="keyword">if</span> isinstance(attention_layer_size, (<span class="built_in">list</span>, tuple)) <span class="keyword">else</span> (attention_layer_size,))</span><br><span class="line">    <span class="keyword">if</span> len(attention_layer_sizes) != len(attention_mechanisms):</span><br><span class="line">        raise <span class="constructor">ValueError(<span class="string">"If provided, attention_layer_size must contain exactly one integer per attention_mechanism, saw: %d vs %d"</span> % (<span class="params">len</span>(<span class="params">attention_layer_sizes</span>)</span>, len(attention_mechanisms)))</span><br><span class="line">    self._attention_layers = tuple(layers_core.<span class="constructor">Dense(<span class="params">attention_layer_size</span>, <span class="params">name</span>=<span class="string">"attention_layer"</span>, <span class="params">use_bias</span>=False, <span class="params">dtype</span>=<span class="params">attention_mechanisms</span>[<span class="params">i</span>].<span class="params">dtype</span>)</span> for i, attention_layer_size <span class="keyword">in</span> enumerate(attention_layer_sizes))</span><br><span class="line">    self._attention_layer_size = sum(attention_layer_sizes)</span><br><span class="line"><span class="keyword">else</span>:</span><br><span class="line">    self._attention_layers = None</span><br><span class="line">    self._attention_layer_size = sum(attention_mechanism.values.get<span class="constructor">_shape()</span><span class="literal">[-<span class="number">1</span>]</span>.value for attention_mechanism <span class="keyword">in</span> attention_mechanisms) </span><br><span class="line">    </span><br><span class="line">for i, attention_mechanism <span class="keyword">in</span> enumerate(self._attention_mechanisms):</span><br><span class="line">    attention, alignments = <span class="constructor">_compute_attention(<span class="params">attention_mechanism</span>, <span class="params">cell_output</span>, <span class="params">previous_alignments</span>[<span class="params">i</span>], <span class="params">self</span>.<span class="params">_attention_layers</span>[<span class="params">i</span>] <span class="params">if</span> <span class="params">self</span>.<span class="params">_attention_layers</span> <span class="params">else</span> None)</span></span><br><span class="line">    alignment_history = previous_alignment_history<span class="literal">[<span class="identifier">i</span>]</span>.write(state.time, alignments) <span class="keyword">if</span> self._alignment_history <span class="keyword">else</span> <span class="literal">()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>alignment_history：即是否将之前的 alignments 存储到 state 中，以便于后期进行可视化展示。</li>
                    <li>cell_input_fn：将 Input 进行处理的方式，默认会将上一步的 Attention 进行 拼接操作，以免造成重复关注同样的内容。代码调用如下：</li>
                  </ul>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cell_inputs = <span class="literal">self</span>._cell_input_fn(inputs, <span class="keyword">state</span>.attention)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>output_attention：是否将 Attention 返回，如果是 False 则返回 Output，否则返回 Attention，默认是 True。</li>
                    <li>initial_cell_state：计算时的初始状态。</li>
                    <li>name：自定义名称。</li>
                  </ul>
                  <p>AttentionWrapper 的核心方法在它的 call() 方法，即类似于 RNNCell 的 call() 方法，AttentionWrapper 类对其进行了重载，代码实现如下：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def call(<span class="literal">self</span>, inputs, <span class="keyword">state</span>):</span><br><span class="line">    <span class="comment"># Step 1</span></span><br><span class="line">    cell_inputs = <span class="literal">self</span>._cell_input_fn(inputs, <span class="keyword">state</span>.attention)</span><br><span class="line">    <span class="comment"># Step 2</span></span><br><span class="line">    cell_state = <span class="keyword">state</span>.cell_state</span><br><span class="line">    cell_output, next_cell_state = <span class="literal">self</span>._cell(cell_inputs, cell_state)</span><br><span class="line">    <span class="comment"># Step 3</span></span><br><span class="line">    if <span class="literal">self</span>._is_multi:</span><br><span class="line">        previous_alignments = <span class="keyword">state</span>.alignments</span><br><span class="line">        previous_alignment_history = <span class="keyword">state</span>.alignment_history</span><br><span class="line">    else:</span><br><span class="line">        previous_alignments = [<span class="keyword">state</span>.alignments]</span><br><span class="line">        previous_alignment_history = [<span class="keyword">state</span>.alignment_history]</span><br><span class="line">    all_alignments = []</span><br><span class="line">    all_attentions = []</span><br><span class="line">    all_histories = []</span><br><span class="line">    <span class="keyword">for</span> i, attention_mechanism <span class="keyword">in</span> enumerate(<span class="literal">self</span>._attention_mechanisms):</span><br><span class="line">        attention, alignments = _compute_attention(attention_mechanism, cell_output, previous_alignments[i], <span class="literal">self</span>._attention_layers[i] if <span class="literal">self</span>._attention_layers else None)</span><br><span class="line">        alignment_history = previous_alignment_history[i].write(<span class="keyword">state</span>.time, alignments) if <span class="literal">self</span>._alignment_history else ()</span><br><span class="line">        all_alignments.append(alignments)</span><br><span class="line">        all_histories.append(alignment_history)</span><br><span class="line">        all_attentions.append(attention)</span><br><span class="line">    <span class="comment"># Step 4</span></span><br><span class="line">    attention = array_ops.concat(all_attentions, <span class="number">1</span>)</span><br><span class="line">    <span class="comment"># Step 5</span></span><br><span class="line">    next_state = AttentionWrapperState(</span><br><span class="line">        time=<span class="keyword">state</span>.time + <span class="number">1</span>,</span><br><span class="line">        cell_state=next_cell_state,</span><br><span class="line">        attention=attention,</span><br><span class="line">        alignments=<span class="literal">self</span>._item_or_tuple(all_alignments),</span><br><span class="line">        alignment_history=<span class="literal">self</span>._item_or_tuple(all_histories))</span><br><span class="line">    <span class="comment"># Step 6</span></span><br><span class="line">    if <span class="literal">self</span>._output_attention:</span><br><span class="line">        return attention, next_state</span><br><span class="line">    else:</span><br><span class="line">        return cell_output, next_state</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里将一些异常判断代码去除了，以便于结构看得更清晰。 首先在第一步中，调用了 _cell_input_fn() 方法，对 inputs 和 state.attention 变量进行处理，默认是使用 concat() 函数拼接，作为当前时间步的输入。因为可能前一步的 Attention 可能对当前 Attention 有帮助，以免让模型连续两次将注意力放在同一个地方。 在第二步中，其实就是调用了普通的 RNNCell 的 call() 方法，得到输出和下一步的状态。 第三步中，这时得到的输出其实并没有用上 AttentionMechanism 中的 alignments 信息，所以当前的输出信息中我们并没有跟 Encoder 的信息做 Attention，所以这里还需要调用 _compute_attention() 方法进行权重的计算，其方法实现如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def <span class="constructor">_compute_attention(<span class="params">attention_mechanism</span>, <span class="params">cell_output</span>, <span class="params">previous_alignments</span>, <span class="params">attention_layer</span>)</span>:</span><br><span class="line">    alignments = attention<span class="constructor">_mechanism(<span class="params">cell_output</span>, <span class="params">previous_alignments</span>=<span class="params">previous_alignments</span>)</span></span><br><span class="line">    expanded_alignments = array_ops.expand<span class="constructor">_dims(<span class="params">alignments</span>, 1)</span></span><br><span class="line">    context = math_ops.matmul(expanded_alignments, attention_mechanism.values)</span><br><span class="line">    context = array_ops.squeeze(context, <span class="literal">[<span class="number">1</span>]</span>)</span><br><span class="line">    <span class="keyword">if</span> attention_layer is not None:</span><br><span class="line">        attention = attention<span class="constructor">_layer(<span class="params">array_ops</span>.<span class="params">concat</span>([<span class="params">cell_output</span>, <span class="params">context</span>], 1)</span>)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        attention = context</span><br><span class="line">    return attention, alignments</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个方法接收四个参数，其中 attention<em>mechanism 就是 AttentionMechanism 的实例，cell_output 就是当前 Output，previous_alignments 是上步的 alignments 信息，调用 attention_mechanism 计算之后就会得到当前步的 alignments 信息了，即 $ \alpha</em>{ij} $。接下来再利用 alignments 信息进行加权运算，得到 attention 信息，即 $ c_{i} $，最后将二者返回。 在第四步中，就是将 attention 结果每个时间步进行 concat，得到 attention vector。 第五步中，声明 AttentionWrapperState 作为下一步的状态。 第六步，判断是否要输出 Attention，如果是，输出 Attention 及下一步状态，否则输出 Outputs 及下一步状态。 好，以上便是整个 AttentionWrapper 源码解析过程，了解了源码之后，再做模型优化的话就非常得心应手了。</p>
                  <h2 id="参考来源"><a href="#参考来源" class="headerlink" title="参考来源"></a><a href="https://github.com/Germey/AI/blob/master/Attention%E5%8E%9F%E7%90%86%E5%8F%8ATensorFlow%20AttentionWrapper%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90.md#%E5%8F%82%E8%80%83%E6%9D%A5%E6%BA%90" target="_blank" rel="noopener"></a>参考来源</h2>
                  <ul>
                    <li><a href="https://arxiv.org/abs/1406.1078" target="_blank" rel="noopener">Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation, Cho et al</a></li>
                    <li><a href="https://arxiv.org/abs/1409.3215" target="_blank" rel="noopener">Sequence to Sequence Learning with Neural Networks, Sutskever et al</a></li>
                    <li><a href="https://arxiv.org/abs/1409.0473" target="_blank" rel="noopener">Neural Machine Translation by Jointly Learning to Align and Translate, Bahdanau et al</a></li>
                    <li><a href="https://arxiv.org/abs/1508.04025" target="_blank" rel="noopener">Effective Approaches to Attention-based Neural Machine Translation, Luong, et al</a></li>
                    <li><a href="https://arxiv.org/abs/1602.07868" target="_blank" rel="noopener">Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks, Salimans, et al</a></li>
                    <li><a href="http://news.ifeng.com/a/20170901/51842411_0.shtml" target="_blank" rel="noopener">http://news.ifeng.com/a/20170901/51842411_0.shtml</a></li>
                    <li><a href="https://blog.csdn.net/qsczse943062710/article/details/79539005" target="_blank" rel="noopener">https://blog.csdn.net/qsczse943062710/article/details/79539005</a></li>
                    <li><a href="https://zhuanlan.zhihu.com/p/34393028" target="_blank" rel="noopener">https://zhuanlan.zhihu.com/p/34393028</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-25 01:41:33" itemprop="dateCreated datePublished" datetime="2018-03-25T01:41:33+08:00">2018-03-25</time>
                </span>
                <span id="/5873.html" class="post-meta-item leancloud_visitors" data-flag-title="Attention原理及TensorFlow AttentionWrapper源码解析" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>14k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>13 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5846.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5846.html" class="post-title-link" itemprop="url">Requests库作者另一神器Pipenv的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2>
                  <p>我们在运行 Python 项目的时候经常会遇到一些版本问题，例如 A 项目依赖于 Django 1.5，而 B 项目又依赖 Django 2.0，而我们的系统却只有一个 Python 解释器，我们所有的包都被装在了 Python 安装目录的 site-packages 目录下，所以 Django 只能是某个特定的版本，所以这样就会导致运行的时候导致 A 或 B 项目出现兼容问题。为了解决这个问题，我们可能会使用 virtualenv 来为项目创建一套独立的 Python 运行环境，或者我们可能会使用 Docker 容器来实现不同项目的隔离运行，但总的来说，它们使用起来其实并没有那么方便。另外在进行 Python 包管理时，requirements.txt 这样的包依赖标识文件也显得很鸡肋，在某些情况下可能会带来一些麻烦。为了解决这些问题，一个更加使用方便的包管理工具诞生了，叫做 Pipenv，接下来就让我们一起来了解一下它的用法。</p>
                  <h2 id="简介"><a href="#简介" class="headerlink" title="简介"></a>简介</h2>
                  <p>Pipenv，它的项目简介为 Python Development Workflow for Humans，是 Python 著名的 requests 库作者 kennethreitz 写的一个包管理工具，它可以为我们的项目自动创建和管理虚拟环境并非常方便地管理 Python 包，现在它也已经是 Python 官方推荐的包管理工具。 Pipenv 我们可以简单理解为 pip 和 virtualenv 的集合体，它可以为我们的项目自动创建和管理一个虚拟环境。virtualenv 在使用时我们需要手动创建一个虚拟环境然后激活，Pipenv 会自动创建。另外我们之前可能使用 requirements.txt 文件来标识项目所需要的依赖，但是这样会带来一些问题，如有的 requirements.txt 中只是将库名列出来了，没有严格指定版本号，这样就可能会导致不同时间安装的库版本是不同的，如 requirements.txt 文件中对 Django 的依赖只写了一个 django，可能在 2016 年的时候运行安装会安装 Django 的 1.x 版本，到了 2017 年就会安装 Django 的 2.x 版本，所以可能导致一些麻烦。为了解决这个问题，Pipenv 直接弃用了 requirements.txt，会同时它会使用一个叫做 Pipfile 和 Pipfile.lock 的文件来管理项目所需的依赖包，而不再是简单地使用 requirements.txt 文件来记录项目所需要的依赖。 总的来说，Pipenv 可以解决如下问题：</p>
                  <ul>
                    <li>我们不需要再手动创建虚拟环境，Pipenv 会自动为我们创建，它会在某个特定的位置创建一个 virtualenv 环境，然后调用 pipenv shell 命令切换到虚拟环境。</li>
                    <li>使用 requirements.txt 可能会导致一些问题，所以 Pipenv 使用 Pipfile 和 Pipfile.lock 来替代之，而且 Pipfile 如果不存在的话会自动创建，而且在安装、升级、移除依赖包的时候会自动更新 Pipfile 和 Pipfile.lock 文件。</li>
                    <li>广泛使用 Hash 校验，保证安全性。</li>
                    <li>可以更清晰地查看 Python 包及其关系，调用 pipenv graph 即可呈现，结果简单明了。</li>
                    <li>可通过自动加载 .env 读取环境变量，简化开发流程。</li>
                  </ul>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <p>本文内容基于 Python 3.6 说明，默认的 Python 解释器命令为 python3，包管理工具命令为 pip3。 Pipenv 是基于 Python 开发的包，所以可以直接用 pip 来安装，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> pipenv</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外还有多种安装方式，如 Pipsi、Nix、Homebrew，安装方式可以参考：<a href="http://pipenv.readthedocs.io/en/latest/#install-pipenv-today" target="_blank" rel="noopener">http://pipenv.readthedocs.io/en/latest/#install-pipenv-today</a>。</p>
                  <h2 id="基本使用"><a href="#基本使用" class="headerlink" title="基本使用"></a>基本使用</h2>
                  <p>首先我们可以新建一个项目，例如叫做 PipenvTest，然后新建一个 Python 脚本，例如叫 main.py，内容为：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import django</span><br><span class="line">print(<span class="name">django</span>.get_version())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>直接用系统的 Python3 运行此脚本：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">python3</span> main.<span class="keyword">py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">1.11</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以看到系统安装的 Django 版本是 1.11。但是我们想要本项目基于 Django 2.x 开发，当然我们可以选择将系统的 Django 版本升级，但这样又可能会影响其他的项目的运行，所以这并不是一个好的选择。为了不影响系统环境的 Django 版本，所以我们可以用 Pipenv 来创建一个虚拟环境。 在该目录下，输入 pipenv 命令即可查看命令的完整用法：</p>
                  <figure class="highlight">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Usage</span>: pipenv [OPTIONS] COMMAND [ARGS]...</span><br><span class="line"></span><br><span class="line"><span class="attribute">Options:</span></span><br><span class="line">  --update         Update Pipenv &amp; pip to latest.</span><br><span class="line">  --where          Output project home information.</span><br><span class="line">  --venv           Output virtualenv information.</span><br><span class="line">  --py             Output Python interpreter information.</span><br><span class="line">  --envs           Output Environment Variable options.</span><br><span class="line">  --rm             Remove the virtualenv.</span><br><span class="line">  --bare           Minimal output.</span><br><span class="line">  --completion     Output completion (to be eval'd).</span><br><span class="line">  --man            Display manpage.</span><br><span class="line">  --three / --two  Use Python 3/2 when creating virtualenv.</span><br><span class="line">  --python TEXT    Specify which version of Python virtualenv should use.</span><br><span class="line">  --site-packages  Enable site-packages for the virtualenv.</span><br><span class="line">  --jumbotron      An easter egg, effectively.</span><br><span class="line">  --version        Show the version and exit.</span><br><span class="line">  -h, --help       Show this message and exit.</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Usage Examples:</span><br><span class="line">   Create a new project using Python 3.6, specifically:</span><br><span class="line">   $ pipenv --python 3.6</span><br><span class="line"></span><br><span class="line">   Install all dependencies for a project (including dev):</span><br><span class="line">   $ pipenv install --dev</span><br><span class="line"></span><br><span class="line">   Create a lockfile containing pre-releases:</span><br><span class="line">   $ pipenv lock --pre</span><br><span class="line"></span><br><span class="line">   Show a graph of your installed dependencies:</span><br><span class="line">   $ pipenv graph</span><br><span class="line"></span><br><span class="line">   Check your installed dependencies for security vulnerabilities:</span><br><span class="line">   $ pipenv check</span><br><span class="line"></span><br><span class="line">   Install a local setup.py into your virtual environment/Pipfile:</span><br><span class="line">   $ pipenv install -e .</span><br><span class="line"></span><br><span class="line"><span class="attribute">Commands:</span></span><br><span class="line">  check      Checks for security vulnerabilities and against PEP 508 markers</span><br><span class="line">             provided in Pipfile.</span><br><span class="line">  graph      Displays currently–installed dependency graph information.</span><br><span class="line">  install    Installs provided packages and adds them to Pipfile, or (if none</span><br><span class="line">             is given), installs all packages.</span><br><span class="line">  lock       Generates Pipfile.lock.</span><br><span class="line">  open       View a given module in your editor.</span><br><span class="line">  run        Spawns a command installed into the virtualenv.</span><br><span class="line">  shell      Spawns a shell within the virtualenv.</span><br><span class="line">  uninstall  Un-installs a provided package and removes it from Pipfile.</span><br><span class="line">  update     Uninstalls all packages, and re-installs package(s) in [packages]</span><br><span class="line">             to latest compatible versions.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们首先验证一下当前的项目是没有创建虚拟环境的，调用如下命令：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--venv</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">No virtualenv has been created <span class="keyword">for</span> <span class="keyword">this</span> <span class="keyword">project</span> yet!</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这说明当前的项目尚未创建虚拟环境，接下来我们利用 Pipenv 来创建一个虚拟环境：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--three</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--python 3.6</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>都可以创建一个 Python3 的虚拟环境，—three 代表创建一个 Python3 版本的虚拟环境，—python 则可以指定特定的 Python 版本，当然 —two 则创建一个 Python2 版本的虚拟环境，但前提你的系统必须装有该版本的 Python 才可以。 执行完毕之后，样例输出如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Warning: the environment variable LANG is not <span class="keyword">set</span>!</span><br><span class="line">We recommend setting this <span class="keyword">in</span> ~/.profile (<span class="keyword">or</span> equivalent) <span class="keyword">for</span> proper expected behavior.</span><br><span class="line">Creating a virtualenv <span class="keyword">for</span> this <span class="keyword">project</span>…</span><br><span class="line"><span class="keyword">Using</span> /usr/<span class="keyword">local</span>/<span class="keyword">bin</span>/python3 <span class="keyword">to</span> <span class="keyword">create</span> virtualenv…</span><br><span class="line">⠋Running virtualenv <span class="keyword">with</span> interpreter /usr/<span class="keyword">local</span>/<span class="keyword">bin</span>/python3</span><br><span class="line"><span class="keyword">Using</span> base prefix <span class="string">'/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6'</span></span><br><span class="line"><span class="keyword">New</span> python executable <span class="keyword">in</span> /<span class="keyword">Users</span>/CQC/.local/<span class="keyword">share</span>/virtualenvs/PipenvTest-VSTVh89E/<span class="keyword">bin</span>/python3<span class="number">.6</span></span><br><span class="line">Also creating executable <span class="keyword">in</span> /<span class="keyword">Users</span>/CQC/.local/<span class="keyword">share</span>/virtualenvs/PipenvTest-VSTVh89E/<span class="keyword">bin</span>/python</span><br><span class="line">Installing setuptools, pip, wheel...done.</span><br><span class="line">Virtualenv location: /<span class="keyword">Users</span>/CQC/.local/<span class="keyword">share</span>/virtualenvs/PipenvTest-VSTVh89E</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里显示 Pipenv 利用 /usr/local/bin/python3 作为 virtualenv 的解释器，然后在 /Users/CQC/.local/share/virtualenvs/PipenvTest-VSTVh89E/bin 目录下创建了一个新的 Python3 解释器，同时还创建了两个可执行文件别名 python3.6 和 python，另外我们还可以发现目录下多了一个 Pipfile 文件，这时虚拟环境就创建完成了。 我们切换到 PipenvTest-VSTVh89E/bin 目录查看一下文件结构，可以看到这里面包含了 pip、pip3、pip3.6、python、python3、python3.6 等可执行文件，实际上目录结构和使用 virtualenv 时是完全一样的，只不过文件夹的位置不同而已。 接下来我们可以切换到该虚拟环境下执行命令，执行如下命令即可：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">shell</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕之后样例输出如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Spawning environment shell (/bin/zsh). <span class="keyword">Use</span> <span class="string">'exit'</span> <span class="keyword">to</span> leave.</span><br><span class="line"><span class="keyword">source</span> /<span class="keyword">Users</span>/CQC/.local/<span class="keyword">share</span>/virtualenvs/PipenvTest-VSTVh89E/<span class="keyword">bin</span>/<span class="keyword">activate</span>                                                            </span><br><span class="line">CQC-MAC% <span class="keyword">source</span> /<span class="keyword">Users</span>/CQC/.local/<span class="keyword">share</span>/virtualenvs/PipenvTest-VSTVh89E/<span class="keyword">bin</span>/<span class="keyword">activate</span></span><br><span class="line">(PipenvTest-VSTVh89E) CQC-MAC%</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实际上这也和 virtualenv 激活的流程一样，也是调用了类似 source venv/bin/activate 方法将这个路径加到全局环境变量最前面，这样就会优先调用该路径下的 python、python3、python3.6 可执行文件了。 这时候我们会发现命令行的样子就变了，前面多了一个 (PipenvTest-VSTVh89E) 的标识，代表当前我们已经切换到了虚拟环境下。 这时我们用 which 或 where 命令查看一下 Python 可执行文件的路径，命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% which python3</span><br><span class="line"><span class="regexp">/Users/</span>CQC<span class="regexp">/.local/</span>share<span class="regexp">/virtualenvs/</span>PipenvTest-VSTVh89E<span class="regexp">/bin/</span>python3</span><br><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% which python3.<span class="number">6</span></span><br><span class="line"><span class="regexp">/Users/</span>CQC<span class="regexp">/.local/</span>share<span class="regexp">/virtualenvs/</span>PipenvTest-VSTVh89E<span class="regexp">/bin/</span>python3.<span class="number">6</span></span><br><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% which python</span><br><span class="line"><span class="regexp">/Users/</span>CQC<span class="regexp">/.local/</span>share<span class="regexp">/virtualenvs/</span>PipenvTest-VSTVh89E<span class="regexp">/bin/</span>python</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现当前的 Python 可执行路径都被切换到了 PipenvTest-VSTVh89E/bin 目录下，调用的是虚拟环境中的 Python 解释器，这时我们重新执行刚才的脚本，命令如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% <span class="keyword">python3</span> main.<span class="keyword">py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时我们可以发现报了如下错误：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Traceback (most recent <span class="keyword">call</span> <span class="keyword">last</span>):</span><br><span class="line">  <span class="keyword">File</span> <span class="string">"main.py"</span>, line <span class="number">1</span>, <span class="keyword">in</span> &lt;<span class="keyword">module</span>&gt;</span><br><span class="line">    <span class="keyword">import</span> django</span><br><span class="line">ModuleNotFoundError: <span class="keyword">No</span> <span class="keyword">module</span> named <span class="string">'django'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这其实是因为新的虚拟环境没有安装任何的 Python 第三方包，实际上如果直接使用 virtualenv 时也是这样的结果。这是因为新的虚拟环境是一个全新的 Python 环境，它默认只包含了 Python 内置的包以及 pip、wheel、setuptools 包，其他的第三方包都没有安装。 这时我们可以使用 Pipenv 来安装 django 包，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">install</span> django</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行后输出结果如下：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">Installing</span> <span class="string">django…</span></span><br><span class="line"><span class="attr">Collecting</span> <span class="string">django</span></span><br><span class="line">  <span class="attr">Downloading</span> <span class="string">Django-2.0.2-py3-none-any.whl (7.1MB)</span></span><br><span class="line"><span class="attr">Collecting</span> <span class="string">pytz (from django)</span></span><br><span class="line">  <span class="attr">Downloading</span> <span class="string">pytz-2018.3-py2.py3-none-any.whl (509kB)</span></span><br><span class="line"><span class="attr">Installing</span> <span class="string">collected packages: pytz, django</span></span><br><span class="line"><span class="attr">Successfully</span> <span class="string">installed django-2.0.2 pytz-2018.3</span></span><br><span class="line"></span><br><span class="line"><span class="attr">Adding</span> <span class="string">django to Pipfile's [packages]…</span></span><br><span class="line"><span class="attr">Locking</span> <span class="string">[dev-packages] dependencies…</span></span><br><span class="line"><span class="attr">Locking</span> <span class="string">[packages] dependencies…</span></span><br><span class="line"><span class="attr">Updated</span> <span class="string">Pipfile.lock (e101fb)!</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果有这样的输出结果就代表成功安装了 Django，可以看到此时安装的 Django 版本为 2.0，代表我们的虚拟环境成功安装了 Django 2.0 版本。 同时我们还注意到它输出了一句话叫做 Updated Pipfile.lock，这时我们可以发现项目路径下又生成了一个 Pipfile.lock 文件，内容如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"_meta"</span>: &#123;</span><br><span class="line">        <span class="attr">"hash"</span>: &#123;</span><br><span class="line">            <span class="attr">"sha256"</span>: <span class="string">"7b9623243d9c22b1f333ee710aff70d0cbcdf1dd7e0aac69230dc76855d27270"</span></span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"host-environment-markers"</span>: &#123;</span><br><span class="line">            <span class="attr">"implementation_name"</span>: <span class="string">"cpython"</span>,</span><br><span class="line">            <span class="attr">"implementation_version"</span>: <span class="string">"3.6.1"</span>,</span><br><span class="line">            <span class="attr">"os_name"</span>: <span class="string">"posix"</span>,</span><br><span class="line">            <span class="attr">"platform_machine"</span>: <span class="string">"x86_64"</span>,</span><br><span class="line">            <span class="attr">"platform_python_implementation"</span>: <span class="string">"CPython"</span>,</span><br><span class="line">            <span class="attr">"platform_release"</span>: <span class="string">"17.4.0"</span>,</span><br><span class="line">            <span class="attr">"platform_system"</span>: <span class="string">"Darwin"</span>,</span><br><span class="line">            <span class="attr">"platform_version"</span>: <span class="string">"Darwin Kernel Version 17.4.0: Sun Dec 17 09:19:54 PST 2017; root:xnu-4570.41.2~1/RELEASE_X86_64"</span>,</span><br><span class="line">            <span class="attr">"python_full_version"</span>: <span class="string">"3.6.1"</span>,</span><br><span class="line">            <span class="attr">"python_version"</span>: <span class="string">"3.6"</span>,</span><br><span class="line">            <span class="attr">"sys_platform"</span>: <span class="string">"darwin"</span></span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"pipfile-spec"</span>: <span class="number">6</span>,</span><br><span class="line">        <span class="attr">"requires"</span>: &#123;&#125;,</span><br><span class="line">        <span class="attr">"sources"</span>: [</span><br><span class="line">            &#123;</span><br><span class="line">                <span class="attr">"name"</span>: <span class="string">"pypi"</span>,</span><br><span class="line">                <span class="attr">"url"</span>: <span class="string">"https://pypi.python.org/simple"</span>,</span><br><span class="line">                <span class="attr">"verify_ssl"</span>: <span class="literal">true</span></span><br><span class="line">            &#125;</span><br><span class="line">        ]</span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attr">"default"</span>: &#123;</span><br><span class="line">        <span class="attr">"django"</span>: &#123;</span><br><span class="line">            <span class="attr">"hashes"</span>: [</span><br><span class="line">                <span class="string">"sha256:af18618ce3291be5092893d8522fe3919661bf3a1fb60e3858ae74865a4f07c2"</span>,</span><br><span class="line">                <span class="string">"sha256:9614851d4a7ff8cbd32b73c6076441f377c45a5bbff7e771798fb02c43c31f47"</span></span><br><span class="line">            ],</span><br><span class="line">            <span class="attr">"version"</span>: <span class="string">"==2.0.2"</span></span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="attr">"pytz"</span>: &#123;</span><br><span class="line">            <span class="attr">"hashes"</span>: [</span><br><span class="line">                <span class="string">"sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe"</span>,</span><br><span class="line">                <span class="string">"sha256:f93ddcdd6342f94cea379c73cddb5724e0d6d0a1c91c9bdef364dc0368ba4fda"</span>,</span><br><span class="line">                <span class="string">"sha256:61242a9abc626379574a166dc0e96a66cd7c3b27fc10868003fa210be4bff1c9"</span>,</span><br><span class="line">                <span class="string">"sha256:ba18e6a243b3625513d85239b3e49055a2f0318466e0b8a92b8fb8ca7ccdf55f"</span>,</span><br><span class="line">                <span class="string">"sha256:07edfc3d4d2705a20a6e99d97f0c4b61c800b8232dc1c04d87e8554f130148dd"</span>,</span><br><span class="line">                <span class="string">"sha256:3a47ff71597f821cd84a162e71593004286e5be07a340fd462f0d33a760782b5"</span>,</span><br><span class="line">                <span class="string">"sha256:5bd55c744e6feaa4d599a6cbd8228b4f8f9ba96de2c38d56f08e534b3c9edf0d"</span>,</span><br><span class="line">                <span class="string">"sha256:887ab5e5b32e4d0c86efddd3d055c1f363cbaa583beb8da5e22d2fa2f64d51ef"</span>,</span><br><span class="line">                <span class="string">"sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0"</span></span><br><span class="line">            ],</span><br><span class="line">            <span class="attr">"version"</span>: <span class="string">"==2018.3"</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attr">"develop"</span>: &#123;&#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到里面标识了 Python 环境基本信息，以及依赖包的版本及 hashes 值。 另外我们还可以注意到 Pipfile 文件内容也有更新，[packages] 部分多了一句 django = “<em>“，标识了本项目依赖于 Django，这个其实类似于 requirements.txt 文件。 那么到这里有小伙伴可能就会问了， Pipfile 和 Pipfile.lock 有什么用呢？ Pipfile 其实一个 TOML 格式的文件，标识了该项目依赖包的基本信息，还区分了生产环境和开发环境的包标识，作用上类似 requirements.txt 文件，但是功能更为强大。Pipfile.lock 详细标识了该项目的安装的包的精确版本信息、最新可用版本信息和当前库文件的 hash 值，顾明思义，它起了版本锁的作用，可以注意到当前 Pipfile.lock 文件中的 Django 版本标识为 ==2.0.2，意思是当前我们开发时使用的就是 2.0.2 版本，它可以起到版本锁定的功能。 举个例子，刚才我们安装了 Django 2.0.2 的版本，即目前（2018.2.27）的最新版本。但可能 Django 以后还会有更新，比如某一天 Django 更新到了 2.1 版本，这时如果我们想要重新部署本项目到另一台机器上，假如此时不存在 Pipfile.lock 文件，只存在 Pipfile文件，由于 Pipfile 文件中标识的 Django 依赖为 django = “</em>“，即没有版本限制，它会默认安装最新版本的 Django，即 2.1，但由于 Pipfile.lock 文件的存在，它会根据 Pipfile.lock 来安装，还是会安装 Django 2.0.2，这样就会避免一些库版本更新导致不兼容的问题。 请记住：任何情况下都不要手动修改 Pipfile.lock 文件！ 好，接下来我们再回归正题，现在已经安装好了 Django 了，那么我们重新运行此脚本便可以成功输出 Django 版本信息了：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% <span class="keyword">python3</span> main.<span class="keyword">py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2.0</span><span class="number">.2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功安装了 Django 2.x 了，和系统的 Django 1.11 没有任何冲突。 在此模式的命令行下，我们就可以使用虚拟环境下的 Python 解释器，而且所安装的依赖包对外部系统没有任何影响，而且使用 Pipfile 和 Pipfile.lock 来管理项目的依赖更加方便和健壮。 如果想要退出虚拟环境，只需要输入 exit 命令即可：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(PipenvTest-VSTVh89E) CQC-MAC% <span class="keyword">exit</span></span><br><span class="line">➜  PipenvTest python3 main.py </span><br><span class="line"><span class="number">1.11</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输入退出命令之后，我们重新再运行此脚本，就会重新使用系统的 Python 解释器，Django 版本又重新回到了 1.11。 由此可以看来，有了 Pipenv，我们可以使用 Pipfile 和 Pipfile.lock 来方便地管理和维护项目的依赖包，而且可以实现虚拟环境运行，避免了包冲突问题，可谓一举两得。</p>
                  <h2 id="常用命令"><a href="#常用命令" class="headerlink" title="常用命令"></a>常用命令</h2>
                  <p>上文我们介绍了 Pipenv 的基本操作，下面我们再介绍一下它的一些常用命令。</p>
                  <h3 id="虚拟环境路径"><a href="#虚拟环境路径" class="headerlink" title="虚拟环境路径"></a>虚拟环境路径</h3>
                  <p>我们可以使用 —venv 参数来获得虚拟环境路径：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--venv</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>样例输出如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="regexp">/Users/</span>CQC<span class="regexp">/.local/</span>share<span class="regexp">/virtualenvs/</span>PipenvTest-VSTVh89E</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见这个路径是一个标准的路径，Pipenv 会把虚拟环境统一放到 virtualenvs 文件夹下，而不是本项目路径下。</p>
                  <h3 id="Python-解释器路径"><a href="#Python-解释器路径" class="headerlink" title="Python 解释器路径"></a>Python 解释器路径</h3>
                  <p>要获取虚拟环境 Python 解释器路径，可以使用 —py 参数：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--py</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>样例输出如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="regexp">/Users/</span>CQC<span class="regexp">/.local/</span>share<span class="regexp">/virtualenvs/</span>PipenvTest-VSTVh89E<span class="regexp">/bin/</span>python</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="加载系统-Python-包"><a href="#加载系统-Python-包" class="headerlink" title="加载系统 Python 包"></a>加载系统 Python 包</h3>
                  <p>默认情况下，新创建的虚拟环境是不包含任何第三方包的，但我们也可以开启加载系统 Python 包功能，使用 —site-packages 即可：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="comment">--site-packages</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样创建的虚拟环境便可以使用系统已安装的 Python 包了。</p>
                  <h3 id="开启虚拟环境"><a href="#开启虚拟环境" class="headerlink" title="开启虚拟环境"></a>开启虚拟环境</h3>
                  <p>要开启虚拟环境只需要执行如下命令：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">shell</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以进入虚拟环境，此时运行的 python、python3 命令都是虚拟环境下的。</p>
                  <h3 id="安装-Python-包"><a href="#安装-Python-包" class="headerlink" title="安装 Python 包"></a>安装 Python 包</h3>
                  <p>安装 Python 包我们不再需要 pip 来安装，直接使用 Pipenv 也可安装，如安装 requests，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">install</span> requests</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成之后会同时更新项目目录下的 Pipfile 和 Pipfile.lock 文件。 有时候一些 Python 包是仅仅开发环境需要的，如 pytest，这时候我们通过添加 —dev 参数即可，命令如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">install</span> pytest <span class="comment">--dev</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时候，pytest 的依赖便会记录在 Pipfile 的 [dev-packages] 区域：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="section">[dev-packages]</span></span><br><span class="line"><span class="attr">pytest</span> = <span class="string">"*"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="获取包依赖"><a href="#获取包依赖" class="headerlink" title="获取包依赖"></a>获取包依赖</h3>
                  <p>我们可以使用命令来清晰地呈现出当前安装的 Python 包版本及之间的依赖关系，命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">pipenv graph</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>样例结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Django==<span class="number">2.0</span><span class="number">.2</span></span><br><span class="line">  - pytz [required: Any, installed: <span class="number">2018.3</span>]</span><br><span class="line">pytest==<span class="number">3.4</span><span class="number">.1</span></span><br><span class="line">  - attrs [required: &gt;=<span class="number">17.2</span><span class="number">.0</span>, installed: <span class="number">17.4</span><span class="number">.0</span>]</span><br><span class="line">  - pluggy [required: &lt;<span class="number">0.7</span>,&gt;=<span class="number">0.5</span>, installed: <span class="number">0.6</span><span class="number">.0</span>]</span><br><span class="line">  - py [required: &gt;=<span class="number">1.5</span><span class="number">.0</span>, installed: <span class="number">1.5</span><span class="number">.2</span>]</span><br><span class="line">  - setuptools [required: Any, installed: <span class="number">38.5</span><span class="number">.1</span>]</span><br><span class="line">  - six [required: &gt;=<span class="number">1.10</span><span class="number">.0</span>, installed: <span class="number">1.11</span><span class="number">.0</span>]</span><br><span class="line">requests==<span class="number">2.18</span><span class="number">.4</span></span><br><span class="line">  - certifi [required: &gt;=<span class="number">2017.4</span><span class="number">.17</span>, installed: <span class="number">2018.1</span><span class="number">.18</span>]</span><br><span class="line">  - chardet [required: &gt;=<span class="number">3.0</span><span class="number">.2</span>,&lt;<span class="number">3.1</span><span class="number">.0</span>, installed: <span class="number">3.0</span><span class="number">.4</span>]</span><br><span class="line">  - idna [required: &lt;<span class="number">2.7</span>,&gt;=<span class="number">2.5</span>, installed: <span class="number">2.6</span>]</span><br><span class="line">  - urllib3 [required: &lt;<span class="number">1.23</span>,&gt;=<span class="number">1.21</span><span class="number">.1</span>, installed: <span class="number">1.22</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到结果非常清晰，Django 当前安装了 2.0.2版本，依赖于 pytz 任何版本，已经安装了 2018.3 版本；pytest 已经安装了 3.4.1 版本，依赖 attrs&gt;=17.2.0 版本，已经安装了 17.4.0 版本，另外还依赖 pluggy、py、setuptools、six 这些库。总之包的依赖关系一目了然。</p>
                  <h3 id="卸载-Python-包"><a href="#卸载-Python-包" class="headerlink" title="卸载 Python 包"></a>卸载 Python 包</h3>
                  <p>卸载 Python 包也非常简单，如卸载 requests 包，命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">pipenv uninstall requests</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>卸载完成之后，Pipfile 和 Pipfile.lock 文件同样会更新。 如果要卸载全部 Python 包，可以添加 —all 参数：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">uninstall</span> <span class="comment">--all</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="产生-Pipfile-lock"><a href="#产生-Pipfile-lock" class="headerlink" title="产生 Pipfile.lock"></a>产生 Pipfile.lock</h3>
                  <p>有时候可能 Pipfile.lock 文件不存在或被删除了，这时候我们可以使用如下命令生成：</p>
                  <figure class="highlight cos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pipenv <span class="keyword">lock</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上便是一些常用的 Pipenv 命令，如果要查看更多用法可以参考其官方文档：<a href="https://docs.pipenv.org/#pipenv-usage" target="_blank" rel="noopener">https://docs.pipenv.org/#pipenv-usage</a>。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本文介绍了 Pipenv 的基本用法，作为 pip 和 virtualenv 的结合体，我们可以利用它更方便地创建和管理 Python 虚拟环境，还可以用更加科学的方式管理 Python 包，一举两得。 嗯，是时候抛弃 virtualenv 和 pip 了！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-15 21:39:41" itemprop="dateCreated datePublished" datetime="2018-03-15T21:39:41+08:00">2018-03-15</time>
                </span>
                <span id="/5846.html" class="post-meta-item leancloud_visitors" data-flag-title="Requests库作者另一神器Pipenv的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>10 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/14/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/14/">14</a><span class="page-number current">15</span><a class="page-number" href="/page/16/">16</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/16/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
